{"cells":[{"metadata":{},"cell_type":"markdown","source":"This is my first public kernetl. Please upvote if you find it useful.\nThanks to https://www.kaggle.com/ragnar123/exploratory-data-analysis-and-baseline for providing the starter code."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np, pandas as pd, os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nimport seaborn as sns, matplotlib.pyplot as plt, matplotlib.patches as patches, plotly.offline as py, plotly.graph_objs as go, plotly.express as px, lightgbm as lgb, plotly.figure_factory as ff, gc, json\nfrom plotly import tools, subplots\npy.init_notebook_mode(connected = True)\npd.set_option('max_columns', 1000)\nfrom bokeh.models import Panel, Tabs\nfrom bokeh.io import output_notebook, show\nfrom bokeh.plotting import figure\nfrom keras.preprocessing import text, sequence\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path = '/kaggle/input/tensorflow2-question-answering/'\ntrain_path = 'simplified-nq-train.jsonl'\ntest_path = 'simplified-nq-test.jsonl'\nsample_submission_path = 'sample_submission.csv'\n\ndef read_data(path, sample = True, chunksize = 10000):\n    if sample == True:\n        df = []\n        with open(path, 'rt') as reader:\n            for i in range(chunksize):\n                df.append(json.loads(reader.readline()))\n        df = pd.DataFrame(df)\n        print('Our sampled dataset have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n    else:\n        df = pd.read_json(path, orient = 'records', lines = True)\n        print('Our dataset have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n        gc.collect()\n    return df\n\ntrain = read_data(path+train_path, sample = True)\ntest = read_data(path+test_path, sample = False)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(path + sample_submission_path)\nprint('Our sample submission has {} rows'.format(sample_submission.shape[0]))\nsample_submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"index=0\nquestion_text_0 = train.loc[index, 'question_text']\nprint('The question is : ', question_text_0)\ndocument_text_0 = train.loc[index, 'document_text'].split()\nprint('Length of wiki article is : ', len(document_text_0))\nlong_answer_candidates_0 = train.loc[index, 'long_answer_candidates']\nprint('Count of long answer candidates is :', len(long_answer_candidates_0))\nannotations_0 = train['annotations'][index][0]\nprint('Ground truth is : ', annotations_0)\n\nif annotations_0['short_answers']!=[]:\n    print('The short answer is : ', \" \".join(document_text_0[annotations_0['short_answers'][0]['start_token']:annotations_0['short_answers'][0]['end_token']]))\nelse:\n    print('Short answer doesn\\'t exist')\nif annotations_0['long_answer']['end_token']>0:\n    print('The long answer is : ', \" \".join(document_text_0[long_answer_candidates_0[annotations_0['long_answer']['candidate_index']]['start_token']:long_answer_candidates_0[annotations_0['long_answer']['candidate_index']]['end_token']]))\nelse:\n    print('Long answer doesn\\'t exist')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"yes_no_answer = []\nfor i in range(len(train)):\n    yes_no_answer.append(train['annotations'][i][0]['yes_no_answer'])\nyes_no_answer = pd.DataFrame({'yes_no_answer': yes_no_answer})\n\nyes_no_answer['yes_no_answer'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_target_variable(df):\n        short_answer = []\n        for i in range(len(df)):\n            short = df['annotations'][i][0]['short_answers']\n            if short == []:\n                yes_no = df['annotations'][i][0]['yes_no_answer']\n                if yes_no == 'NO' or yes_no == 'YES':\n                    short_answer.append([yes_no, -1, -1. -1])\n                else:\n                    short_answer.append(['EMPTY', -1, -1, -1])\n            else:\n                short = short[0]\n                st = short['start_token']\n                et = short['end_token']\n                short_answer.append([f'{st}'+':'+f'{et}', st, et, et-st])\n        short_answer = pd.DataFrame(short_answer, columns=['short_answer', 'short_start', 'short_end', 'short_length'])\n        \n        long_answer= []\n        for i in range(len(df)):\n            long = df['annotations'][i][0]['long_answer']\n            if long['start_token'] == -1:\n                long_answer.append(['EMPTY',-1,-1, -1])\n            else:\n                st = long['start_token']\n                et = long['end_token']\n                long_answer.append([f'{st}'+':'+f'{et}',st,et, et-st])\n        long_answer = pd.DataFrame(long_answer, columns=['long_answer', 'long_start', 'long_end', 'long_length'])\n        answer = pd.concat([long_answer, short_answer], axis=1)\n        return answer\n\nanswer = extract_target_variable(train)\nanswer.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answer['diff_start'] = answer['short_start'] - answer['long_start']\nanswer['diff_end'] = answer['long_end'] - answer['short_end']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answer.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n, bins, patches = plt.hist(x=answer[answer['long_length']>-1]['long_length'], bins=1000)\nplt.grid(axis='y')\nplt.xlabel('Length of Long Answer')\nplt.ylabel('Frequency')\nplt.xscale('log')\n#Majority of long answers have length between 30 and 200","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n, bins, patches = plt.hist(x=answer[answer['short_length']>-1]['short_length'], bins=125)\nplt.grid(axis='y')\nplt.xlabel('Length of Short Answer')\nplt.ylabel('Frequency')\nplt.xscale('log')\n#Majority of short answers have length less than 20","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n, bins, patches = plt.hist(x=answer[(answer['short_length']>-1)&(answer['long_length']>-1)]['diff_start'], bins=1000)\nplt.grid(axis='y')\nplt.xlabel('Difference in start of short and long answer')\nplt.ylabel('Frequency')\nplt.xscale('log')\n#Majority of different in start of short and long answers is less than 20","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n, bins, patches = plt.hist(x=answer[(answer['short_length']>-1)&(answer['long_length']>-1)]['diff_end'], bins=1000)\nplt.grid(axis='y')\nplt.xlabel('Difference in end of short and long answer')\nplt.ylabel('Frequency')\nplt.xscale('log')\n#Majority of difference in start of short and long answers is less than 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answer[answer['short_answer']=='EMPTY'].shape[0]/answer.shape[0] #Percent of cases where short answer is empty","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answer[answer['long_answer']=='EMPTY'].shape[0]/answer.shape[0] #Percent of cases where long answer is empty","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answer[(answer['short_answer']!='EMPTY') & (answer['long_answer']=='EMPTY')].shape[0]/answer.shape[0] #Percent of cases where long answer is empty but short answer is not","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"answer[(answer['short_answer']=='EMPTY') & (answer['long_answer']!='EMPTY')].shape[0]/answer.shape[0] #Percent of cases where short answer is empty but long answer is not","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.scatter(answer[(answer['short_length']>-1)&(answer['long_length']>-1)]['long_length'], answer[(answer['short_length']>-1)&(answer['long_length']>-1)]['short_length'], s=1)\nplt.xscale('log')\nplt.yscale('log')\nplt.xlabel('Long Length')\nplt.ylabel('Short Length')\nplt.show()\nprint(np.corrcoef(answer[(answer['short_length']>-1)&(answer['long_length']>-1)]['long_length'], answer[(answer['short_length']>-1)&(answer['long_length']>-1)]['short_length'])[0][1])\n#No correlation between length of long answer and short answer","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}