{"cells":[{"metadata":{},"cell_type":"markdown","source":"# TL;DR\n\n* Most of the data that has very long answer's document is the article like `List_of_...`\n    * `long_answer` is the span of long list in the document\n    * `short_answer` is the one of the cell in the list\n* Some documents (`document_text`) are linked to multiple questions (`question_text`)\n    * linked questions are unique\n    * but some questions have the same `long_answer` span"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nimport matplotlib.pyplot as plt\nfrom tqdm import tqdm_notebook as tqdm","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Check data keys"},{"metadata":{"trusted":true},"cell_type":"code","source":"from itertools import islice\n\nnq_train_jsonl = \"/kaggle/input/tensorflow2-question-answering/simplified-nq-train.jsonl\"\n\nwith open(nq_train_jsonl, \"r\") as f:\n    for line in islice(f, 1):\n        train =json.loads(line)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['annotations']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['annotations'][0]['long_answer']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Retrive very long answer data\nRetrive train data that contains `long_answer` which have very long token (>10000)"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_list = []\nwith open(nq_train_jsonl, \"r\") as f:\n    for line in tqdm(f):\n        data = json.loads(line)\n        long_ans = data['annotations'][0]['long_answer']\n        if long_ans['end_token'] - long_ans['start_token'] > 10000:\n            data_list.append(data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(data_list)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Save results"},{"metadata":{"trusted":true},"cell_type":"code","source":"# save\nwith open(\"./very-long-answer-nq-train.jsonl\", \"w\")as f:\n    for l in data_list:\n        json.dump(l, f)\n        f.write(\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Convert to DataFrame"},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame.from_dict(data_list)\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# annotations data to column\ndf[\"yes_no_answer\"] = df[\"annotations\"].apply(lambda q: q[0][\"yes_no_answer\"])\n\ndf[\"long_answer_end\"] = df[\"annotations\"].apply(lambda q: q[0][\"long_answer\"][\"end_token\"])\ndf[\"long_answer_start\"] = df[\"annotations\"].apply(lambda q: q[0][\"long_answer\"][\"start_token\"])\ndf[\"long_answer_length\"] = df.loc[:,\"long_answer_end\":\"long_answer_start\"].diff(axis=1)[\"long_answer_start\"].abs()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def apply_ans_end(entry):\n    if len(entry)==0:\n        return None\n    return  entry[0][\"end_token\"]\n\ndef apply_ans_start(entry):\n    if len(entry)==0:\n        return None\n    return  entry[0][\"start_token\"]\n\ndf[\"short_answers\"] = df[\"annotations\"].apply(lambda q: q[0][\"short_answers\"])\ndf[\"short_answer_end\"] = df[\"short_answers\"].apply(apply_ans_end)\ndf[\"short_answer_start\"] = df[\"short_answers\"].apply(apply_ans_start)\ndf[\"short_answer_length\"] = df.loc[:,\"short_answer_end\":\"short_answer_start\"].diff(axis=1)[\"short_answer_start\"].abs()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# question_text\ndf[\"head_word\"] = df[\"question_text\"].apply(lambda q:q.split()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## View very long answer data"},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.core.display import HTML","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['document_url'][0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This `document_text` is [List of the highest major summits of North America](https://en.wikipedia.org//w/index.php?title=List_of_the_highest_major_summits_of_North_America&oldid=835916791) at Wikipendia\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_answer_text(s, is_short=False):\n    if is_short:\n        beg = int(s['short_answer_start'])\n        end = int(s['short_answer_end'])\n    else:\n        beg = int(s['long_answer_start'])\n        end = int(s['long_answer_end'])\n        \n    if beg is not None and end is not None:\n        return \" \".join(s['document_text'].split(\" \")[beg:end])\n    else:\n        return None","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### long answer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# long answer\nHTML(get_answer_text(df.iloc[0]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### short answer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# short answer\nHTML(get_answer_text(df.iloc[0], True))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA\n## short answer"},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"short_answer_length\"].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"short_answer_length\"].plot.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"short_answer_length\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## yes_no_answer"},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"yes_no_answer\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## max short_answer_length"},{"metadata":{"trusted":true},"cell_type":"code","source":"max_idx = df[\"short_answer_length\"].idxmax()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df.iloc[max_idx])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df.iloc[max_idx]['document_url'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Electron configurations of the elements (data page) : https://en.wikipedia.org//w/index.php?title=Electron_configurations_of_the_elements_(data_page)"},{"metadata":{"trusted":true},"cell_type":"code","source":"df.iloc[max_idx]['question_text']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### short answer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# short ans\nHTML(get_answer_text(df.iloc[max_idx], True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"get_answer_text(df.iloc[max_idx], True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### long answer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# long ans \nHTML(get_answer_text(df.iloc[max_idx], False))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## max long_answer_length"},{"metadata":{"trusted":true},"cell_type":"code","source":"max_long = df[\"long_answer_length\"].idxmax()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.iloc[max_long]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df.iloc[max_long]['document_url'])\nprint(df.iloc[max_long]['question_text'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"List of Xbox 360 games https://en.wikipedia.org//w/index.php?title=List_of_Xbox_360_games\n\ndivided into two pages now\n* https://en.wikipedia.org/wiki/List_of_Xbox_360_games_(A–L)\n* https://en.wikipedia.org/wiki/List_of_Xbox_360_games_(M–Z)"},{"metadata":{},"cell_type":"markdown","source":"### long answer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# long ans \nHTML(get_answer_text(df.iloc[max_long]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# count question_text\ncount word"},{"metadata":{"trusted":true},"cell_type":"code","source":"df[\"question_text\"].str.split()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\ncnt = Counter()\nfor item in df[\"question_text\"].str.split().to_list():\n    cnt.update(item)\ncnt.most_common(50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# count head word of question_text "},{"metadata":{"trusted":true},"cell_type":"code","source":"# count head word\nplt.figure(figsize=(10,8),dpi=100)\nplt.rcParams[\"font.size\"] = 6\n\ndf.head_word.value_counts().plot(kind='bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head_word.value_counts().head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"`list`, `xbox` ,`cities` is not *interrogative word* (what, when, who, how, which)."},{"metadata":{},"cell_type":"markdown","source":"## head word: list"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.set_option(\"display.max_colwidth\", 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df[df[\"head_word\"]==\"list\"][[\"question_text\", \"document_url\", \"long_answer_start\", \"long_answer_end\"]] \\\n.sort_values(by=[\"document_url\", \"long_answer_start\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## head word: xbox"},{"metadata":{"trusted":true},"cell_type":"code","source":"df[df[\"head_word\"]==\"xbox\"][[\"question_text\", \"document_url\", \"long_answer_start\", \"long_answer_end\"]] \\\n.sort_values(by=[\"document_url\", \"long_answer_start\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## head word: cities"},{"metadata":{"trusted":true},"cell_type":"code","source":"df[df[\"head_word\"]==\"cities\"][[\"question_text\", \"document_url\", \"long_answer_start\", \"long_answer_end\"]] \\\n.sort_values(by=[\"document_url\", \"long_answer_start\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# count document_url like \"List_of_xx\""},{"metadata":{"trusted":true},"cell_type":"code","source":"df['document_url'].str.contains('=List_of_', regex=True).describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## nunique"},{"metadata":{"trusted":true},"cell_type":"code","source":"df['document_url'].nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df['question_text'].nunique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.groupby(['document_url']) \\\n.count()[['long_answer_length', 'short_answer_length']] \\\n.sort_values(by = 'long_answer_length', ascending=False) \\\n.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.groupby(['document_url']).count()['long_answer_length'].hist(bins=19, grid=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.groupby(['document_url']).count()['long_answer_length'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" * one document, one question : 254/340 = 75% \n * one document, multiple question : 25% \n \n Some documents are linked to multiple questions.\n \n ***\n\n \n Note:\n \n * If you considered about Wikipedia Revision history (like `&oldid=XXX`) , more documents are linked to multiple questions.\n     * For example, [List_of_Xbox_360_games_compatible_with_Xbox_One](https://en.wikipedia.org//w/index.php?title=List_of_Xbox_360_games_compatible_with_Xbox_One) is old version of [List_of_backward_compatible_games_for_Xbox_One](https://en.wikipedia.org//w/index.php?title=List_of_backward_compatible_games_for_Xbox_One)\n     * https://en.wikipedia.org/w/index.php?title=List_of_backward_compatible_games_for_Xbox_One&action=history"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}