{"cells":[{"metadata":{},"cell_type":"markdown","source":"I check below:\n\n- question_text\n- annotations\n  - short answer\n  - long answer\n\nI use whole 'light_train.csv' data which exclude document_text because it's too large.\nthe 'light_train.csv' is available at https://www.kaggle.com/kentaronakanishi/tfqaconverteddata"},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# basic\nimport os\nimport sys\nfrom typing import Optional, List, Dict\nfrom pathlib import Path\nimport json\nimport pickle\n\n# data\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"datadir_name = 'tfqaconverteddata'\ndatadir = Path('../input/') / datadir_name\noutputdir = Path('../output/')\noutputdir.mkdir()\ntrain_file = 'simplified-nq-train.jsonl'\ntest_file = 'simplified-nq-test.jsonl'","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"%%time\ntrain_df = pd.read_csv(datadir / 'light_train.csv')\ntrain_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\ntest_df = pd.read_json(datadir / test_file, orient='records', lines=True)\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_df.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## check question_text\nWhat I check:\n- word length statistics of question text\n- char length statistics of question text\n- word count in all question text\n  - frequent words\n  - frequent head words (important for question)\n  - head word count accumulation rate"},{"metadata":{"trusted":false},"cell_type":"code","source":"question_texts = train_df['question_text'].values.tolist()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"question_texts[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check length","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"word_lens_list = []\nchar_lens_list = []\n\nfor text in question_texts:\n    word_lens_list.append(len(text.split()))\n    char_lens_list.append(len(text))\nword_lens = np.array(word_lens_list)\nchar_lens = np.array(char_lens_list)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"word_lens.max(), word_lens.min(), word_lens.mean(), word_lens.std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# word_lens histgram: most of questions are shorter than 15\nsns.distplot(word_lens, bins=26, kde=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"char_lens.max(), char_lens.min(), char_lens.mean(), char_lens.std()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# char_lens histgram: most sentences are around 50\nsns.distplot(char_lens, bins=85, kde=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# some data have just 100 char length, check this\nlong_questions = np.array(question_texts)[char_lens == 100]\nlen(long_questions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"long_questions[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check words\nfrom collections import Counter","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# make all word array\nwords = []\nfor text in question_texts:\n    words.extend(text.split())\nall_count = Counter(words).most_common()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# see top 20\nall_count[:20]","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"# see bottom 50, some are misspelling and special symbols and abbreviation\nall_count[-50:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# make all word array\nhead_words = []\nfor text in question_texts:\n    head_words.append(text.split()[0])\nhead_count = Counter(head_words).most_common()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# see top 20\nhead_count[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"total_question = 307373\naccm = []\ncounts = [c for _, c in head_count]\nfor i, (w, cnt) in enumerate(head_count):\n    accm.append(cnt + sum(counts[:i]))\naccm = np.array(accm) / np.sum(counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"plt.title('Accumulate Ratio for word count')\nplt.plot(range(len(head_count[:1000])), accm[:1000])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# top 20 get 80% of head words\nlist(zip(head_count[:20], accm[:20]))\n# who, what, when occupy 50%, and about 70% by where, how ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# - ['who', 'who\\'s', 'what', 'when', 'where', 'how', 'which', 'why', 'what\\'s'] or not","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## check annotation\nWhat I check:\n- annotations column detail\n- short answers\n  - yes/no answer\n  - short answers count\n    - statistics\n    - when yes/no answer exists\n  - length of short answers\n  - whether multiple short answer is same or not\n- long answers\n  - candidates num\n  - target long answer existing num\n  - long answers length\n  - top level count\n  - correct top level count\n  - all answers count"},{"metadata":{"trusted":false},"cell_type":"code","source":"train_df['annotations'].iloc[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# one record is a str of list of json dict\n# there are keys:\nkeys = ['yes_no_answer', 'long_answer', 'short_answers', 'annotation_id']\n# long_answer has keys:\nlong_answer_keys = ['start_token', 'candidate_index', 'end_token']\n# short_answers is a list of dict and keys:\nshort_answers_keys = ['start_token', 'end_token']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# make DataFrame for annotation records\ndef annotations2value(x: str, key: str) -> Optional[str]:\n    if isinstance(x, str):\n        x = x.replace('\\'', '\"')\n    x = json.loads(x)\n    if isinstance(x, list):\n        assert len(x) == 1, f'{x} has multi answers'\n        x = x[0]\n    v = x[key] if key in x else None\n    return v\n\n\ndef short2value(x: List[Dict[str, int]], key: str) -> Optional[List[int]]:\n    if len(x) == 0:\n        return ''\n    return ','.join([str(xx[key]) for xx in x])\n\nfor k in keys:\n    train_df[k] = train_df['annotations'].apply(lambda x: annotations2value(x, k))\n    \nfor k in long_answer_keys:\n    train_df['long_' + k] = train_df['long_answer'].apply(lambda x: x[k])\n    \nfor k in short_answers_keys:\n    train_df['short_' + k] = train_df['short_answers'].apply(lambda x: short2value(x, k))\n    \ndef get_short_count(x: str):\n    if x == '':\n        return 0\n    x = x.split(',')\n    return len(x)\n\ntrain_df['short_answer_count'] = train_df['short_start_token'].apply(get_short_count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# del train_df['annotations'], train_df['long_answer'], train_df['short_answers']\ntrain_df.head()\n# there are some some values -1 in long","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# save annotation DataFrame\n# train_df.to_csv(outputdir / 'annotations_detail.csv', index=False)\nannotation_df = train_df\ndel train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## check short answer\n- yes/no answer\n- short answers count\n- statistics\n- when yes/no answer exists\n- length of short answers\n- whether multiple short answer is same or not"},{"metadata":{"trusted":false},"cell_type":"code","source":"annotation_df = pd.read_csv(outputdir / 'annotations_detail.csv')\nannotation_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check yes_no_answer\nsns.countplot(x=annotation_df['yes_no_answer'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# most of answer is None, check only yes_no\nsns.countplot(x=annotation_df[annotation_df['yes_no_answer'] != 'NONE']['yes_no_answer'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# next check the number of short answer\nprint(annotation_df['short_answer_count'].describe())\nsns.countplot(x=annotation_df['short_answer_count'])\n# short answer has 25 answers at max but most of them are 1 or 2, or 0.\n# 2/3 of short answers are None.\n# what means for multiple short answer? check it later.","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# what value is in short_start/end when answer is yes/no?\nsns.countplot(x=annotation_df[annotation_df['yes_no_answer'] != 'NONE']['short_answer_count'])\n# it refer to zero (None). if the answer is yes/no, there are no short_answer by token index.","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# how many records are zero short answer?\nannotation_df.query(\"yes_no_answer == 'NONE' and short_answer_count == 0\")['example_id'].count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check length of short answer","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check short_start_token if or not it has anomaly","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"short_start_tokens = []\nshort_end_tokens = []\nshort_token_lengths = []\nfor start_token, end_token in annotation_df[['short_start_token', 'short_end_token']].values:\n    if start_token is np.nan and end_token is np.nan:\n        continue\n    short_start_tokens.extend([int(t) for t in start_token.split(',')])\n    short_end_tokens.extend([int(t) for t in end_token.split(',')])\nshort_start_tokens = np.array(short_start_tokens)\nshort_end_tokens = np.array(short_end_tokens)\nshort_token_lengths = short_end_tokens - short_start_tokens","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"short_df = pd.DataFrame(\n    zip(short_start_tokens, short_end_tokens, short_token_lengths),\n    columns=['short_start_tokens', 'short_end_tokens', 'short_token_lengths']\n)\nprint(short_df['short_start_tokens'].describe())\nprint(short_df['short_end_tokens'].describe())\nprint(short_df['short_token_lengths'].describe())\nsns.countplot(x=short_df[short_df['short_token_lengths'] < 20]['short_token_lengths'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# max length of short answer is 250, is it anomaly value?\nprint('more than 20:', short_df[short_df['short_token_lengths'] >= 20]['short_token_lengths'].count())\nprint('more than 50:', short_df[short_df['short_token_lengths'] >= 50]['short_token_lengths'].count())\nprint('more than 100:', short_df[short_df['short_token_lengths'] >= 100]['short_token_lengths'].count())\nprint('more than 150:', short_df[short_df['short_token_lengths'] >= 150]['short_token_lengths'].count())\nprint('more than 200:', short_df[short_df['short_token_lengths'] >= 200]['short_token_lengths'].count())\nprint('more than 220:', short_df[short_df['short_token_lengths'] >= 220]['short_token_lengths'].count())\n# There are 35 samples whose length are longer than 100, what is short\n# check this not short short answer later","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# multiple short answer is same phrase?\n# check length of each multiple short answer\nnot_same_count = 0\nfor start_token, end_token in annotation_df[['short_start_token', 'short_end_token']].values:\n    if start_token is np.nan and end_token is np.nan:\n        continue\n    starts = np.array([int(t) for t in start_token.split(',')])\n    ends = np.array([int(t) for t in end_token.split(',')])\n    if len(starts) <= 1:\n        continue\n    diff = ends - starts\n    is_all_same = (diff == diff[0]).all()\n    if not is_all_same:\n        not_same_count += 1\n\nprint(not_same_count)\n\n# almost multiple answers are not same, I have to check the text later.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## check long answers\n\n- candidates num\n- target long answer existing num\n- long answers length\n- top level count\n- correct top level count"},{"metadata":{"trusted":false},"cell_type":"code","source":"# long answer\nannotation_df = pd.read_csv(outputdir / 'annotations_detail.csv')\nannotation_df.head()","execution_count":null,"outputs":[]},{"metadata":{"scrolled":true,"trusted":false},"cell_type":"code","source":"# long answers have candidates and one of them are correct\n# addition to that, it has 'top_level' column which indicate the answer is included in another answer.\nannotation_df['long_answer_candidates'].iloc[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# firstly, check the number of candidates\nla_keys = ['start_token', 'end_token', 'top_level']\ndef la2value(key, x):\n    if isinstance(x, str):\n        x = x.replace('\\'', '\"')\n        x = x.replace('True', '1')\n        x = x.replace('False', '0')\n    x = json.loads(x)\n    values = []\n    for candidate in x:\n        values.append(int(candidate[key]))\n    values = ','.join([str(v) for v in values])\n    return values\nfor key in la_keys:\n    print('start:', key)\n    annotation_df['long_' + key + 's'] = annotation_df['long_answer_candidates'].apply(lambda x: la2value(key, x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"annotation_df.to_csv(outputdir / 'annotations_detail.csv', index=False)\nannotation_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# firstly check long answer candidates num\nannotation_df['long_candidates_num'] = annotation_df['long_start_tokens'].apply(lambda x: len(x.split(',')))\nprint(annotation_df['long_candidates_num'].describe())\n_ = plt.figure(figsize=(20, 10))\n# sns.countplot(x=annotation_df['long_candidates_num'])\nsns.countplot(x=annotation_df[annotation_df['long_candidates_num'] < 150]['long_candidates_num'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check the record that has no long answer. token == -1 means no long answer\nannotation_df['no_long_answer'] = annotation_df['long_start_token'] == -1\nprint(annotation_df['no_long_answer'].describe())\nsns.countplot(x=annotation_df['no_long_answer'])\n# half of data have no long answer...","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# in case there is no long answer, there shoud be no short answer. check this.\nannotation_df[annotation_df['no_long_answer']]['short_answer_count'].describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# so then, there are\n# - 307373 record\n# - 196649 have no short answer\n# - 155225 have no long answer (off course no short answer if no long answer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# check top_level\nannotation_df['long_top_level_1_num'] = annotation_df['long_top_levels'].apply(lambda x: sum([int(xx) for xx in x.split(',')]))\nannotation_df['long_top_level_0_num'] = annotation_df['long_top_levels'].apply(lambda x: len(x.split(',')) - sum([int(xx) for xx in x.split(',')]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"_ = plt.figure(figsize=(18, 4))\n# sns.countplot(x=annotation_df['long_top_level_0_num'])\nsns.countplot(x=annotation_df[annotation_df['long_top_level_0_num'] <= 100]['long_top_level_0_num'])\n_ = plt.figure(figsize=(18, 4))\n# sns.countplot(x=annotation_df['long_top_level_1_num'])\nsns.countplot(x=annotation_df[annotation_df['long_top_level_1_num'] <= 100]['long_top_level_1_num'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# by histgram\nsns.distplot(annotation_df[annotation_df['long_top_level_0_num'] <= 100]['long_top_level_0_num'], bins=100)\nsns.distplot(annotation_df[annotation_df['long_top_level_1_num'] <= 100]['long_top_level_1_num'], bins=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"num0 = annotation_df['long_top_level_0_num'].sum()\nnum1 = annotation_df['long_top_level_1_num'].sum()\nsns.barplot(y=[num0, num1], x=['0', '1'])\n# about 2/3 data are top_level = 0","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# annotation_df['long_correct_top_level']\nlong_correct_top_level = np.zeros(annotation_df.values.shape[0])\nfor i, target in enumerate(annotation_df['long_candidate_index'].values):\n    long_correct_top_level[i] = annotation_df['long_top_levels'].iloc[i].split(',')[target]\nannotation_df['long_correct_top_level'] = long_correct_top_level","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sns.countplot(x=annotation_df['long_correct_top_level'])\n# most of data are top_level = 1 for only correct data\n# even though 2/3 data are top_level = 0 plotted above","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# next, check long answer length\nannotation_df['long_correct_length'] = annotation_df['long_end_token'] - annotation_df['long_start_token']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"print(annotation_df[annotation_df['long_correct_length'] >= 1]['long_correct_length'].describe())\n_ = plt.figure(figsize=(20, 6))\n_ = sns.distplot(annotation_df.query('long_correct_length > 1 & long_correct_length < 1000')['long_correct_length'], bins=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# all long candidates\nlong_start_tokens = []\nlong_end_tokens = []\nlong_token_lengths = []\nfor start_token, end_token in annotation_df[['long_start_tokens', 'long_end_tokens']].values:\n    long_start_tokens.extend([int(t) for t in start_token.split(',')])\n    long_end_tokens.extend([int(t) for t in end_token.split(',')])\nlong_start_tokens = np.array(long_start_tokens)\nlong_end_tokens = np.array(long_end_tokens)\nlong_token_lengths = long_end_tokens - long_start_tokens","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"long_df = pd.DataFrame(\n    long_token_lengths,\n    columns=['long_token_lengths']\n)\ndel long_start_tokens, long_end_tokens, long_token_lengths","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# print(long_df['long_start_tokens'].describe())\n# print(long_df['long_end_tokens'].describe())\nprint(long_df['long_token_lengths'].describe())","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"_ = plt.figure(figsize=(20, 6))\n_ = sns.distplot(\n    long_df.query('long_token_lengths < 1000')['long_token_lengths'],\n    bins=100,\n    label='all_long_candidate_answer'\n)\n_ = sns.distplot(\n    annotation_df.query('long_correct_length > 1 & long_correct_length < 1000')['long_correct_length'],\n    bins=100,\n    label='correct_long_answer'\n)\n_ = plt.legend()\n# big difference in length distribution of candidates and correct answer ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"annotation_df.to_csv(outputdir / 'annotations_detail.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# todo\n# - multiple short answer text\n# - not short short answer\n# - answer check by question type","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.5"}},"nbformat":4,"nbformat_minor":1}