{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## 6.7 数据集分析","metadata":{}},{"cell_type":"code","source":"#P6.1 导入库\nimport numpy as np \nimport pandas as pd \nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport plotly.offline as py  #需要安装plotly\nimport plotly.graph_objs as go\nimport gc\nimport json\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport os","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:32:28.435935Z","iopub.execute_input":"2022-12-05T07:32:28.436671Z","iopub.status.idle":"2022-12-05T07:32:29.598327Z","shell.execute_reply.started":"2022-12-05T07:32:28.436530Z","shell.execute_reply":"2022-12-05T07:32:29.597179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\n\npath = '/kaggle/input/tensorflow2-question-answering'\n# data_path = '../input/mlcoursechapter3/chapter3/dataset'\ndata_path = '/kaggle/input/tensorflow2-question-answering'\nsys.path.append(path)\nprint(path)","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:32:29.600846Z","iopub.execute_input":"2022-12-05T07:32:29.601664Z","iopub.status.idle":"2022-12-05T07:32:29.611808Z","shell.execute_reply.started":"2022-12-05T07:32:29.601615Z","shell.execute_reply":"2022-12-05T07:32:29.609954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# P6.2 显示数据文件名称\nfilenames = os.listdir(f'{path}')\nfor filename in filenames:\n    print(filename)","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:32:29.613810Z","iopub.execute_input":"2022-12-05T07:32:29.614362Z","iopub.status.idle":"2022-12-05T07:32:29.629557Z","shell.execute_reply.started":"2022-12-05T07:32:29.614315Z","shell.execute_reply":"2022-12-05T07:32:29.627345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#P6.3 读取训练集前30000行数据，读取测试集全部数据\npath = f'{path}/'\ntrain_path = 'simplified-nq-train.jsonl'\ntest_path = 'simplified-nq-test.jsonl'\nsample_submission_path = 'sample_submission.csv'\n\ndef read_data(path, sample = True, chunksize = 30000):\n    if sample == True:\n        df = []\n        with open(path, 'rt') as reader:\n            for i in range(chunksize):\n                df.append(json.loads(reader.readline()))\n        df = pd.DataFrame(df)\n        print('抽样的数据包含 {} 行，{} 列'.format(df.shape[0], df.shape[1]))\n    else:\n        df = pd.read_json(path, orient = 'records', lines = True)\n        print('数据集包含 {} 行， {} 列'.format(df.shape[0], df.shape[1]))\n        gc.collect()\n    return df\n\ntrain = read_data(path+train_path, sample = True)  #抽样训练集数据\ntest = read_data(path+test_path, sample = False)  #测试集数据\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:32:29.632979Z","iopub.execute_input":"2022-12-05T07:32:29.634422Z","iopub.status.idle":"2022-12-05T07:33:11.426963Z","shell.execute_reply.started":"2022-12-05T07:32:29.634367Z","shell.execute_reply":"2022-12-05T07:33:11.425414Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# P6.4 显示提交数据文件内容\nsample_submission = pd.read_csv(path + sample_submission_path)\nprint('提交数据集包含 {} 行'.format(sample_submission.shape[0]))\nsample_submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:11.428661Z","iopub.execute_input":"2022-12-05T07:33:11.429013Z","iopub.status.idle":"2022-12-05T07:33:11.453673Z","shell.execute_reply.started":"2022-12-05T07:33:11.428982Z","shell.execute_reply":"2022-12-05T07:33:11.452381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# P6.5 检查训练集缺失值\ndef missing_values(df):\n    df = pd.DataFrame(df.isnull().sum()).reset_index()\n    df.columns = ['features', 'n_missing_values']\n    return df\nmissing_values(train)","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:11.455142Z","iopub.execute_input":"2022-12-05T07:33:11.455669Z","iopub.status.idle":"2022-12-05T07:33:11.502207Z","shell.execute_reply.started":"2022-12-05T07:33:11.455635Z","shell.execute_reply":"2022-12-05T07:33:11.501126Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.7 测试集缺失值\nmissing_values(test)","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:11.504901Z","iopub.execute_input":"2022-12-05T07:33:11.505291Z","iopub.status.idle":"2022-12-05T07:33:11.519865Z","shell.execute_reply.started":"2022-12-05T07:33:11.505260Z","shell.execute_reply":"2022-12-05T07:33:11.518344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.8 训练集问题观察，以索引序号为45的问题为例\nquestion_text= train.loc[1, 'question_text']\nquestion_text","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:11.521431Z","iopub.execute_input":"2022-12-05T07:33:11.521892Z","iopub.status.idle":"2022-12-05T07:33:11.543189Z","shell.execute_reply.started":"2022-12-05T07:33:11.521856Z","shell.execute_reply":"2022-12-05T07:33:11.541742Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.9 训练集文本观察\ndocument_text = train.loc[1, 'document_text'].split()\n\" \".join(document_text[:800])","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:11.545200Z","iopub.execute_input":"2022-12-05T07:33:11.546019Z","iopub.status.idle":"2022-12-05T07:33:11.556991Z","shell.execute_reply.started":"2022-12-05T07:33:11.545981Z","shell.execute_reply":"2022-12-05T07:33:11.555424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.10 备选长答案观察\nlong_answer_candidates= train.loc[1, 'long_answer_candidates']\nlong_answer_candidates[0:10]","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:11.561547Z","iopub.execute_input":"2022-12-05T07:33:11.562032Z","iopub.status.idle":"2022-12-05T07:33:11.572386Z","shell.execute_reply.started":"2022-12-05T07:33:11.561996Z","shell.execute_reply":"2022-12-05T07:33:11.571202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.11 annotation数据观察\nannotations = train['annotations'][1][0]\nannotations","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:11.573962Z","iopub.execute_input":"2022-12-05T07:33:11.574414Z","iopub.status.idle":"2022-12-05T07:33:11.587329Z","shell.execute_reply.started":"2022-12-05T07:33:11.574372Z","shell.execute_reply":"2022-12-05T07:33:11.585790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('提出的问题是 : ', question_text)\nprint('短答案是 : ', \" \".join(document_text[annotations['short_answers'][0]['start_token']:annotations['short_answers'][0]['end_token']]))\nprint('长答案是 : ', \" \".join(document_text[annotations['long_answer']['start_token']:annotations['long_answer']['end_token']]))","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:11.589044Z","iopub.execute_input":"2022-12-05T07:33:11.589657Z","iopub.status.idle":"2022-12-05T07:33:11.598767Z","shell.execute_reply.started":"2022-12-05T07:33:11.589617Z","shell.execute_reply":"2022-12-05T07:33:11.597401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.12 Yes/No类型问题分布\nyes_no_answer = []\nfor i in range(len(train)):\n    yes_no_answer.append(train['annotations'][i][0]['yes_no_answer'])\nyes_no_answer = pd.DataFrame({'yes_no_answer': yes_no_answer})\n    \ndef bar_plot(df, column, title, width, height, n, get_count = True):\n    if get_count == True:\n        cnt_srs = df[column].value_counts(normalize = True)[:n]\n    else:\n        cnt_srs = df\n        \n    trace = go.Bar(\n        x = cnt_srs.index,\n        y = cnt_srs.values,\n        marker = dict(\n            color = '#1E90FF',\n        ),\n        opacity=0.7,\n    )\n\n    layout = go.Layout(\n        title = go.layout.Title(\n            text = title,\n            x = 0.5\n        ),\n        font = dict(size = 14),\n        width = width,\n        height = height,\n        \n    )\n\n    data = [trace]\n    fig = go.Figure(data = data, layout = layout)\n    py.iplot(fig, filename = 'bar_plot')\nbar_plot(yes_no_answer, 'yes_no_answer', 'Yes No Answer Distribution', 400, 400, 3)","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:11.600554Z","iopub.execute_input":"2022-12-05T07:33:11.600939Z","iopub.status.idle":"2022-12-05T07:33:13.092975Z","shell.execute_reply.started":"2022-12-05T07:33:11.600905Z","shell.execute_reply":"2022-12-05T07:33:13.091247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# P6.13 短答案数据提取\ndef extract_target_variable(df, short = True):\n    if short:\n        short_answer = []\n        for i in range(len(df)):\n            short = df['annotations'][i][0]['short_answers']\n            if short == []:\n                yes_no = df['annotations'][i][0]['yes_no_answer']\n                if yes_no == 'NO' or yes_no == 'YES':\n                    short_answer.append(yes_no)\n                else:\n                    short_answer.append('EMPTY')\n            else:\n                short = short[0]\n                st = short['start_token']\n                et = short['end_token']\n                short_answer.append(f'{st}'+':'+f'{et}')\n        short_answer = pd.DataFrame({'short_answer': short_answer})\n        return short_answer\n    else:\n        long_answer = []\n        for i in range(len(df)):\n            long = df['annotations'][i][0]['long_answer']\n            if long['start_token'] == -1:\n                long_answer.append('EMPTY')\n            else:\n                st = long['start_token']\n                et = long['end_token']\n                long_answer.append(f'{st}'+':'+f'{et}')\n        long_answer = pd.DataFrame({'long_answer': long_answer})\n        return long_answer\n        \nshort_answer = extract_target_variable(train)\nshort_answer.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:13.095197Z","iopub.execute_input":"2022-12-05T07:33:13.095650Z","iopub.status.idle":"2022-12-05T07:33:13.499528Z","shell.execute_reply.started":"2022-12-05T07:33:13.095615Z","shell.execute_reply":"2022-12-05T07:33:13.498149Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.14 绘图显示短答案数据分布\nshort_answer['type'] = short_answer['short_answer'].copy()\nshort_answer.loc[(short_answer['short_answer']!='EMPTY') & (short_answer['short_answer']!='YES') & (short_answer['short_answer']!='NO'), 'type'] =  'TEXT'\nbar_plot(short_answer, 'type', 'Short Answer Distribution', 400, 400, 10)","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:13.501489Z","iopub.execute_input":"2022-12-05T07:33:13.502726Z","iopub.status.idle":"2022-12-05T07:33:13.563096Z","shell.execute_reply.started":"2022-12-05T07:33:13.502677Z","shell.execute_reply":"2022-12-05T07:33:13.559954Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.15 长答案数据提取，绘图显示长答案分布\nlong_answer = extract_target_variable(train, False)\nlong_answer.head()","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:13.565551Z","iopub.execute_input":"2022-12-05T07:33:13.566771Z","iopub.status.idle":"2022-12-05T07:33:13.828331Z","shell.execute_reply.started":"2022-12-05T07:33:13.566708Z","shell.execute_reply":"2022-12-05T07:33:13.826908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"long_answer['type'] = long_answer['long_answer'].copy()\nlong_answer.loc[(long_answer['long_answer']!='EMPTY'), 'type'] =  'TEXT'\nbar_plot(long_answer, 'type', 'Long Answer Distribution', 400, 400, 5)","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:13.830373Z","iopub.execute_input":"2022-12-05T07:33:13.831284Z","iopub.status.idle":"2022-12-05T07:33:13.880283Z","shell.execute_reply.started":"2022-12-05T07:33:13.831244Z","shell.execute_reply":"2022-12-05T07:33:13.878918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n#P6.16 统计训练集问题文本中出现频率最高的前20个单词\ndef count_word_frequency(series, top = 0, bot = 20, title = 'Question Text Word Frequency Distribution'):\n    cv = CountVectorizer()   \n    cv_fit = cv.fit_transform(series)    \n    word_list = cv.get_feature_names(); \n    count_list = cv_fit.toarray().sum(axis=0)\n    frequency = pd.DataFrame({'Word': word_list, 'Frequency': count_list})\n    frequency.sort_values(['Frequency'], ascending = False, inplace = True)\n    frequency['Percentage'] = frequency['Frequency']/frequency['Frequency'].sum()\n    frequency.drop('Frequency', inplace = True, axis = 1)\n    frequency['Percentage'] = frequency['Percentage'].round(3)\n    frequency = frequency.iloc[top:bot]\n    frequency.set_index('Word', inplace = True)\n    bar_plot(pd.Series(frequency['Percentage']), 'Percentage', title, 800, 500, 20, False)\n    return frequency\n    \nfrequency = count_word_frequency(train['question_text'],0 , 20,'Train:Question Text Word Frequency Distribution top 20')","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:13.881926Z","iopub.execute_input":"2022-12-05T07:33:13.883080Z","iopub.status.idle":"2022-12-05T07:33:17.819838Z","shell.execute_reply.started":"2022-12-05T07:33:13.883037Z","shell.execute_reply":"2022-12-05T07:33:17.818053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.17 统计训练集问题文本中出现频率排名为第21-40的单词\nfrequency = count_word_frequency(train['question_text'], 20, 40,'Train:Question Text Word Frequency Distribution top 20-39')","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:17.821716Z","iopub.execute_input":"2022-12-05T07:33:17.822247Z","iopub.status.idle":"2022-12-05T07:33:20.746437Z","shell.execute_reply.started":"2022-12-05T07:33:17.822198Z","shell.execute_reply":"2022-12-05T07:33:20.745367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.18 统计测试集问题文本中出现频率排名前20的单词\nfrequency = count_word_frequency(test['question_text'],0,20,'Test:Question Text Word Frequency Distribution top 20')","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:20.748122Z","iopub.execute_input":"2022-12-05T07:33:20.748991Z","iopub.status.idle":"2022-12-05T07:33:20.799720Z","shell.execute_reply.started":"2022-12-05T07:33:20.748951Z","shell.execute_reply":"2022-12-05T07:33:20.798156Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.19 统计测试集问题文本中出现频率排名为第21-40的单词\nfrequency = count_word_frequency(test['question_text'], 20, 40,'Test:Question Text Word Frequency Distribution top 20-39')","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:20.801587Z","iopub.execute_input":"2022-12-05T07:33:20.802405Z","iopub.status.idle":"2022-12-05T07:33:20.857306Z","shell.execute_reply.started":"2022-12-05T07:33:20.802354Z","shell.execute_reply":"2022-12-05T07:33:20.855930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#P6.20 统计测试集正文文本中出现频率排名前20的单词\nfrequency = count_word_frequency(test['document_text'],0,20,'Test:Document Text Word Frequency Distribution top 20')","metadata":{"execution":{"iopub.status.busy":"2022-12-05T07:33:20.859692Z","iopub.execute_input":"2022-12-05T07:33:20.861215Z","iopub.status.idle":"2022-12-05T07:33:23.753943Z","shell.execute_reply.started":"2022-12-05T07:33:20.861157Z","shell.execute_reply":"2022-12-05T07:33:23.752584Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}