{"cells":[{"metadata":{},"cell_type":"markdown","source":"TensorFlow 2.0 Question Answering Competition Notebook\n==================\n* Develop a more effective and robust QA system"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport json\n\ndef read_lines_m(path, max_limit=4000):\n    rlm = []; ml = max_limit\n    for l in open(path, 'r'):\n        rlm.append(json.loads(l))\n        ml -= 1\n        if ml <= 0: break\n    return pd.DataFrame(rlm)\n\np = '../input/tensorflow2-question-answering/'\ntrain = read_lines_m(p + 'simplified-nq-train.jsonl')\ntrain['D'] = [t[0]['long_answer']['start_token'] for t in train.annotations]\ntrain = train[train['D']>-1].reset_index(drop=True)\ntest = read_lines_m(p + 'simplified-nq-test.jsonl').reset_index(drop=True)\nsub = pd.read_csv(p + 'sample_submission.csv')\ntrain.shape, test.shape, sub.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"i =99\nprint('URL:', train.document_url[i])\nprint(train.question_text[i])\nprint(train.long_answer_candidates[i][0])\nprint(' '.join(train.document_text[i].split()[train.annotations[i][0]['long_answer']['start_token'] : train.annotations[i][0]['long_answer']['end_token']]))\nif len(train.annotations[i][0]['short_answers']) > 0:\n    print(' '.join(train.document_text[i].split()[train.annotations[i][0]['short_answers'][0]['start_token'] : train.annotations[i][0]['short_answers'][0]['end_token']]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"la=[t[0]['long_answer']['end_token'] - t[0]['long_answer']['start_token'] for t in train.annotations]\nsa=[t[0]['short_answers'][0]['end_token'] - t[0]['short_answers'][0]['start_token'] for t in train.annotations if len(t[0]['short_answers'])>0]\nnp.median(la), np.median(sa)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from bs4 import BeautifulSoup as b\nfrom nltk.corpus import stopwords\nimport random, nltk\n\ndef qa_word_match(q,a):\n    q = q.lower().split()\n    q = [q1 for q1 in q if q1 not in list(set(stopwords.words('english')))]\n    tm = 0\n    a2 = a[0]\n    for a1 in a:\n        m = np.sum([1 for w in a1.lower().split() if w in q])\n        if m > tm:\n            tm = int(m)\n            a2 = str(a1)\n    return a2\n\nresult = []\nfor i in range(len(test.example_id)):\n    s = b(test.document_text[i], 'html.parser')\n    p = [p.get_text() for p in s.find_all('p', text=True) if len(p.get_text()) > 50]\n    if len(p)>0:\n        a = qa_word_match(test.question_text[i], p)\n        r = test.document_text[i].find(a)\n        r = len(test.document_text[i][:r].split()) - 1\n        long_answer= ''.join([str(r),':', str(r + len(p[0].split()) + 2)])\n    else:\n        try:\n            r = random.randrange(390, len(test.document_text[i].split()))\n        except:\n            r=7\n        long_answer= ''.join([str(r),':', str(r + 114)])\n\n    if len([q for q in ['am', 'are', 'can', 'could', 'did', 'do', 'does', 'has', 'have', 'is', 'may', 'should', 'was', 'were', 'will'] if q in test.question_text[i].lower().split()])>0:\n        short_answer=random.choice(['YES','NO'])\n    else:\n        r = random.randrange(r, r + 114)\n        short_answer=''.join([str(r),':', str(r + 2)])\n    result.append([test.example_id[i] + '_long', long_answer])\n    result.append([test.example_id[i] + '_short', short_answer])\npd.DataFrame(result,columns=['example_id', 'PredictionString']).to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}