{"cells":[{"metadata":{},"cell_type":"markdown","source":"# TL;DR\nThe dataset is **TOO LARGE** for the Kaggle Notebook RAM to load at once."},{"metadata":{},"cell_type":"markdown","source":"# 0. Preparation"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gc\nimport json\nimport subprocess\n\nfrom tqdm import tqdm_notebook as tqdm\nimport matplotlib.pyplot as plt\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DIR = '../input/tensorflow2-question-answering/'\nPATH_TRAIN = DIR + 'simplified-nq-train.jsonl'\nPATH_TEST = DIR + 'simplified-nq-test.jsonl'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 0-1. Number of samples in train & test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"N_TRAIN_bytes = subprocess.check_output('wc -l {}'.format(PATH_TRAIN), shell=True)\nN_TEST_bytes = subprocess.check_output('wc -l {}'.format(PATH_TEST), shell=True)\n\nN_TRAIN = int(N_TRAIN_bytes.split()[0])\nN_TEST = int(N_TEST_bytes.split()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(N_TRAIN)\nprint(N_TEST)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 1. Load .jsonl file iteratively"},{"metadata":{},"cell_type":"markdown","source":"As we know, one of the most common way to convert .jsonl file into pd.DataFrame is  `pd.read_json(FILENAME, orient='records', lines=True)`:"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test = pd.read_json(PATH_TEST, orient='records', lines=True, dtype={'example_id':np.dtype('object')})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"However, since we have a **HUGE train dataset** for this competition, Kaggle Notebook RAM cannot afford this method.\nInstead, we probablly have to load the train dataset iteratively:"},{"metadata":{"trusted":true},"cell_type":"code","source":"json_train_head = []\nN_HEAD = 10\n\nwith open(PATH_TRAIN, 'rt') as f:\n    for i in range(N_HEAD):\n        json_train_head.append(json.loads(f.readline()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_head = pd.DataFrame(json_train_head)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_head","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_head.iloc[0,:].loc['long_answer_candidates']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train_head.iloc[0,:].loc['annotations']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del df_train_head, df_test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Data Visualization"},{"metadata":{},"cell_type":"markdown","source":"## 2-1. Obtain data"},{"metadata":{},"cell_type":"markdown","source":"We must be cautious that **\"short answer\" for this competition corresponds to \"yes-no answer\" in the original dataset**.  "},{"metadata":{"trusted":true},"cell_type":"code","source":"n_long_candidates_train = np.zeros(N_TRAIN)\nt_long_train = np.zeros((N_TRAIN,2))\nt_yesno_train = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(PATH_TRAIN, 'rt') as f:\n    for i in tqdm(range(N_TRAIN)):\n        dic = json.loads(f.readline())\n        n_long_candidates_train[i] = len(dic['long_answer_candidates'])\n        t_long_train[i,0] = dic['annotations'][0]['long_answer']['start_token']\n        t_long_train[i,1] = dic['annotations'][0]['long_answer']['end_token']\n        t_yesno_train.append(dic['annotations'][0]['yes_no_answer'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_long_candidates_test = np.zeros(N_TEST)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(PATH_TEST, 'rt') as f:\n    for i in tqdm(range(N_TEST)):\n        dic = json.loads(f.readline())\n        n_long_candidates_test[i] = len(dic['long_answer_candidates'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2-2. Visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.style.use('seaborn-darkgrid')\nplt.style.use('seaborn-poster')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2-2-1. Number of long answer candidates"},{"metadata":{},"cell_type":"markdown","source":"Some of data for long answers are swamped with a lot of candidates (**7946 in maximum!**):"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(n_long_candidates_train).describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(n_long_candidates_test).describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(n_long_candidates_train, bins=64, alpha=0.5, color='c', label='train')\nplt.xlabel('long answer candidates')\nplt.ylabel('samples')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(n_long_candidates_train[n_long_candidates_train < np.max(n_long_candidates_test)], density=True, bins=64, alpha=0.5, color='c', label='train')\nplt.hist(n_long_candidates_test, density=True, bins=64, alpha=0.5, color='orange', label='test')\nplt.xlabel('long answer candidates')\nplt.ylabel('sample proportion')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2-2-2. Yes-no answer labels"},{"metadata":{},"cell_type":"markdown","source":"We can see significant class imbalance in yes-no answer labels."},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(t_yesno_train, bins=[0,1,2,3], align='left', density=True, rwidth=0.6, color='lightseagreen', label='train')\nplt.xlabel('yes-no answer')\nplt.ylabel('sample proportion')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 2-2-3. Long answer labels"},{"metadata":{},"cell_type":"markdown","source":"Description of start token labels:"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(t_long_train[:,0]).describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Desciption of end token labels:"},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.Series(t_long_train[:,1]).describe()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see below that nearly half of the long answers have start/end token -1.  \nIn other words, there are a considerable number of '**NO ANSWERS**' in long answer labels, not only in yes-no labels:"},{"metadata":{"trusted":true},"cell_type":"code","source":"print('{0:.1f}% of start tokens are -1.'.format(np.sum(t_long_train[:,0] < 0) / N_TRAIN * 100))\nprint('{0:.1f}% of end tokens are -1.'.format(np.sum(t_long_train[:,1] < 0) / N_TRAIN * 100))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"If the start token is -1, the corresponding end token is also -1:"},{"metadata":{"trusted":true},"cell_type":"code","source":"np.sum(t_long_train[:,0] * t_long_train[:,1] < 0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The heatmap below tells us that:\n- when the start token and/or the end token are -1, yes-no answer is 'NONE'\n- yes-no answer 'NONE' does not always mean that the start token and/or the end token are -1"},{"metadata":{"trusted":true},"cell_type":"code","source":"# no_answer_state[1,:] is the number of train data whose start token and end token are -1\n# no_answer_state[:,1] is the number of train data whose yes-no answer is 'NONE'\n\nno_answer_state = np.zeros((2,2))\nno_answer_state[1,1] = np.sum((t_long_train[:,0]==-1) * (np.array([ 1 if t=='NONE' else 0 for t in t_yesno_train ])))\nno_answer_state[1,0] = np.sum((t_long_train[:,0]==-1) * (np.array([ 0 if t=='NONE' else 1 for t in t_yesno_train ])))\nno_answer_state[0,1] = np.sum((t_long_train[:,0]>=0) * (np.array([ 1 if t=='NONE' else 0 for t in t_yesno_train ])))\nno_answer_state[0,0] = np.sum((t_long_train[:,0]>=0) * (np.array([ 0 if t=='NONE' else 1 for t in t_yesno_train ])))                             ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"no_answer_state","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.heatmap(no_answer_state / N_TRAIN, annot=True, annot_kws={\"size\": 25}, fmt='.3f', vmin=0, vmax=1, cmap='Blues_r')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del n_long_candidates_train, n_long_candidates_test, t_long_train, t_yesno_train, no_answer_state\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 3. Text Word Counts"},{"metadata":{},"cell_type":"markdown","source":"Let us look into word counts of question texts & document texts."},{"metadata":{},"cell_type":"markdown","source":"### 3-1. Obtain data"},{"metadata":{"trusted":true},"cell_type":"code","source":"q_lens_train = np.zeros(N_TRAIN)\nd_lens_train = np.zeros(N_TRAIN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(PATH_TRAIN, 'rt') as f:\n    for i in tqdm(range(N_TRAIN)):\n        dic = json.loads(f.readline())\n        q_lens_train[i] = len(dic['question_text'].split())\n        d_lens_train[i] = len(dic['document_text'].split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"q_lens_test = np.zeros(N_TEST)\nd_lens_test = np.zeros(N_TEST)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with open(PATH_TEST, 'rt') as f:\n    for i in tqdm(range(N_TEST)):\n        dic = json.loads(f.readline())\n        q_lens_test[i] = len(dic['question_text'].split())\n        d_lens_test[i] = len(dic['document_text'].split())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### 3-2. Visualization"},{"metadata":{},"cell_type":"markdown","source":"#### 3-2-1. Word counts of question text"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(q_lens_train, density=True, bins=8, alpha=0.5, color='c', label='train')\nplt.hist(q_lens_test, density=True, bins=8, alpha=0.5, color='orange', label='test')\nplt.xlabel('question length')\nplt.ylabel('sample proportion')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 3-2-2. Word counts of document text"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(d_lens_train, density=True, bins=64, alpha=0.5, color='c', label='train')\nplt.hist(d_lens_test, density=True, bins=64, alpha=0.5, color='orange', label='test')\nplt.xlabel('document length')\nplt.ylabel('sample proportion')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let us have fun!  \nComments and recommendations will be welcomed ;)"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}