{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\nimport numpy as np \nimport pandas as pd\nimport re\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom sklearn.metrics import accuracy_score, f1_score\nfrom tqdm import tqdm_notebook as tqdm\nfrom Levenshtein import ratio as levenshtein_distance\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction import text\n\nfrom scipy import spatial","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_answers = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"html_tags = ['<P>', '</P>', '<Table>', '</Table>', '<Tr>', '</Tr>', '<Ul>', '<Ol>', '<Dl>', '</Ul>', '</Ol>', \\\n             '</Dl>', '<Li>', '<Dd>', '<Dt>', '</Li>', '</Dd>', '</Dt>']\nr_buf = ['is', 'are', 'do', 'does', 'did', 'was', 'were', 'will', 'can', 'the', 'a', 'of', 'in', 'and', 'on', \\\n         'what', 'where', 'when', 'which'] + html_tags\n\ndef clean(x):\n    x = x.lower()\n    for r in r_buf:\n        x = x.replace(r, '')\n    x = re.sub(' +', ' ', x)\n    return x\n\nbin_question_tokens = ['is', 'are', 'do', 'does', 'did', 'was', 'were', 'will', 'can']\nstop_words = text.ENGLISH_STOP_WORDS.union([\"book\"])\n\ndef predict(json_data, annotated=False):\n    # Parse JSON data\n    candidates = json_data['long_answer_candidates']\n    candidates = [c for c in candidates if c['top_level'] == True]\n    doc_tokenized = json_data['document_text'].split(' ')\n    question = json_data['question_text']\n    question_s = question.split(' ') \n    if annotated:\n        ann = json_data['annotations'][0]\n\n    # TFIDF for the document\n    tfidf = TfidfVectorizer(ngram_range=(1,1), stop_words=stop_words)\n    tfidf.fit([json_data['document_text']])\n    q_tfidf = tfidf.transform([question]).todense()\n\n    # Find the nearest answer from candidates\n    distances = []\n    scores = []\n    i_ann = -1\n    p_cnt = 1\n    for i, c in enumerate(candidates):\n        s, e = c['start_token'], c['end_token']\n        t = ' '.join(doc_tokenized[s:e])\n        distances.append(levenshtein_distance(clean(question), clean(t)))\n        \n        t_tfidf = tfidf.transform([t]).todense()\n        score = 1 - spatial.distance.cosine(q_tfidf, t_tfidf)\n        \n        # See this kernel https://www.kaggle.com/petrov/first-long-paragraph\n        if doc_tokenized[s] == '<P>':\n            score += 0.25**p_cnt\n            p_cnt += 1\n        \n#         score = 0\n        \n#         for w in doc_tokenized[s:e]:\n#             if w in q_s:\n#                 score += 0.1\n\n        scores.append(score)\n#     print(scores)\n    # Format results\n#     ans = candidates[np.argmin(distances)]\n#     ans = candidates[np.argmax(scores)]\n    ans = (np.array(candidates)[np.argsort(scores)])[-n_answers:].tolist()\n    \n    if np.max(scores) < 0.2:\n        ans_long = ['-1:-1']\n        ans = [{'start_token': 0, 'end_token': 0}]\n    else:\n#         ans_long = str(ans['start_token']) + ':' + str(ans['end_token'])\n        ans_long = [str(a['start_token']) + ':' + str(a['end_token']) for a in ans]\n    if question_s[0] in bin_question_tokens:\n        ans_short = 'YES'\n    else:\n        ans_short = ''\n        \n    # Preparing data for debug\n    if annotated:\n        ann_long_text = ' '.join(doc_tokenized[ann['long_answer']['start_token']:ann['long_answer']['end_token']])\n        if ann['yes_no_answer'] == 'NONE':\n            if len(json_data['annotations'][0]['short_answers']) > 0:\n                ann_short_text = ' '.join(doc_tokenized[ann['short_answers'][0]['start_token']:ann['short_answers'][0]['end_token']])\n            else:\n                ann_short_text = ''\n        else:\n            ann_short_text = ann['yes_no_answer']\n    else:\n        ann_long_text = ''\n        ann_short_text = ''\n        \n    ans_long_text = [' '.join(doc_tokenized[a['start_token']:a['end_token']]) for a in ans]\n    if len(ans_short) > 0 or ans_short == 'YES':\n        ans_short_text = ans_short\n    else:\n        ans_short_text = '' # Fix when short answers will work\n                    \n    return ans_long, ans_short, question, ann_long_text, ann_short_text, ans_long_text, ans_short_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nids = []\nanns = []\npreds = []\n\n# Debug data\nquestions = []\nann_texts = []\nans_texts = []\n\nn_samples = 500\n\nwith open('/kaggle/input/tensorflow2-question-answering/simplified-nq-train.jsonl', 'r') as json_file:\n    cnt = 0\n    for line in tqdm(json_file):\n        json_data = json.loads(line)\n\n        l_ann = str(json_data['annotations'][0]['long_answer']['start_token']) + ':' + \\\n            str(json_data['annotations'][0]['long_answer']['end_token'])\n        if json_data['annotations'][0]['yes_no_answer'] == 'NONE':\n            if len(json_data['annotations'][0]['short_answers']) > 0:\n                s_ann = str(json_data['annotations'][0]['short_answers'][0]['start_token']) + ':' + \\\n                    str(json_data['annotations'][0]['short_answers'][0]['end_token'])\n            else:\n                s_ann = ''\n        else:\n            s_ann = json_data['annotations'][0]['yes_no_answer']\n\n        l_ans, s_ans, question, ann_long_text, ann_short_text, ans_long_text, ans_short_text = predict(json_data, annotated=True)\n        \n        ids += [str(json_data['example_id']) + '_long']*len(l_ans)\n        ids.append(str(json_data['example_id']) + '_short')\n        \n        anns += [l_ann]*len(l_ans)\n        anns.append(s_ann)\n        \n        preds += l_ans\n        preds.append(s_ans)\n        questions += [question]*len(l_ans)\n        questions.append(question)\n        ann_texts += [ann_long_text]*len(l_ans)\n        ann_texts.append(ann_short_text)\n        ans_texts += ans_long_text\n        ans_texts.append(ans_short_text)\n        \n        cnt += 1\n        if cnt >= n_samples:\n            break\n        \ntrain_ann = pd.DataFrame()\ntrain_ann['example_id'] = ids\ntrain_ann['question'] = questions\ntrain_ann['CorrectString'] = anns\ntrain_ann['CorrectText'] = ann_texts\nif len(preds) > 0:\n    train_ann['PredictionString'] = preds\n    train_ann['PredictionText'] = ans_texts\n    \ntrain_ann.to_csv('train_data.csv', index=False)\ntrain_ann.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Should be replaced by code from https://github.com/google-research-datasets/natural-questions/blob/master/nq_eval.py\nf1 = f1_score(train_ann['CorrectString'].values, train_ann['PredictionString'].values, average='micro')\nprint(f'F1-score: {f1:.4f}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nids = []\nanns = []\npreds = []\n\n# Debug data\nquestions = []\nann_texts = []\nans_texts = []\n\nwith open('/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl', 'r') as json_file:\n    cnt = 0\n    for line in tqdm(json_file):\n        json_data = json.loads(line)\n        \n        l_ans, s_ans, question, ann_long_text, ann_short_text, ans_long_text, ans_short_text = predict(json_data)\n\n        ids += [str(json_data['example_id']) + '_long']*len(l_ans)\n        ids.append(str(json_data['example_id']) + '_short')\n        preds += l_ans\n        preds.append(s_ans)\n        questions += [question]*len(l_ans)\n        questions.append(question)\n        ans_texts += ans_long_text\n        ans_texts.append(ans_short_text)\n         \n#         cnt += 1\n#         if cnt >= n_samples:\n#             break\n        \nsubm = pd.DataFrame()\nsubm['example_id'] = ids\nsubm['question'] = questions\nsubm['PredictionString'] = preds\nsubm['PredictionText'] = ans_texts\nsubm.to_csv('test_data.csv', index=False)\n\ng = subm[['example_id', 'PredictionString']].groupby('example_id').agg(lambda x: ' '.join(x) if len(x) > 1 else x).reset_index()\ng.to_csv('submission.csv', index=False)\n\nsubm.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}