{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import gc\nimport json\nimport numpy as np \nimport pandas as pd\nimport re\nfrom tqdm import tqdm_notebook as tqdm\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def readjson(train,QorD):\n    with open('/kaggle/input/tensorflow2-question-answering/simplified-nq-'+train+'.jsonl', 'r') as json_file:\n        cnt = 0\n        temp=[]\n        tempdoc=[]\n        for line in tqdm(json_file,total=100):\n            json_data = json.loads(line) \n            # Collect annotations\n            if train=='train':\n                start = json_data['annotations'][0]['long_answer']['start_token']\n                end = json_data['annotations'][0]['long_answer']['end_token']\n\n                # Collect short annotations\n                if json_data['annotations'][0]['yes_no_answer'] == 'NONE':\n                    if len(json_data['annotations'][0]['short_answers']) > 0:\n                        sans = str(json_data['annotations'][0]['short_answers'][0]['start_token']) + ':' + \\\n                            str(json_data['annotations'][0]['short_answers'][0]['end_token'])\n                    else:\n                        sans = ''\n                else:\n                    sans = json_data['annotations'][0]['yes_no_answer']\n        \n                if QorD=='d':\n                    tempdocline=[json_data['document_text'][:5000]]\n                else:\n                    templine=[start,end,sans,json_data['question_text']]\n\n            else:\n                if QorD=='d':\n                    tempdocline=[json_data['document_text'] ]\n                else:\n                    templine=[json_data['question_text'] ]\n            if QorD=='d':\n                tempdoc.append(tempdocline)\n            else:\n                temp.append(templine)\n        if QorD=='q':\n            filenm=train+'_question_text.csv'\n            pd.DataFrame(temp).to_csv(filenm)\n            del temp\n        else:\n            filenm=train+'split_document_text.csv'        \n            pd.DataFrame( wiki_tagsplit(pd.DataFrame(tempdoc)[0],0)).to_csv(filenm)\n            \n    return\n\n#  find the start stop  token position of a sentence \ndef find_start_end_token(txt,sent):\n    pos=txt.find(np.str(sent) )\n    start=len(txt[:pos-1].split(' '))\n    end=start+len(np.str(sent).split(' '))\n    return start-1,end-1\n\nfrom bs4 import BeautifulSoup\n\ndef wiki_tagsplit(html,wi):\n    temp=[]\n    #html = trainW.iloc[wi]['wiki'] \n    for hi in html:\n        \n        soup = BeautifulSoup(hi, 'html.parser')\n        for ti in ['h1','h2','p','table','tr']:  #splitting tags extracting this features\n            allep=soup.find_all(ti) #p paragraph\n            for pi in allep:\n                start,stop=find_start_end_token(hi,pi.get_text())\n                if start>1:\n                    line=[wi,ti,start,stop,pi.get_text()]\n                    temp.append(line)\n                    \n        wi=wi+1\n    return pd.DataFrame(temp,columns=['id','tag','start','stop','txt'])\n\nreadjson('test','q')\nreadjson('test','d')\n\nreadjson('train','q')\nreadjson('train','d')\n\n#readjson('train','document_text')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subm=pd.read_csv('../input/tensorflow2-question-answering/sample_submission.csv')\nsubm.to_csv('submit.csv')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}