{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-16T13:35:10.823778Z","iopub.execute_input":"2022-02-16T13:35:10.824588Z","iopub.status.idle":"2022-02-16T13:35:10.869503Z","shell.execute_reply.started":"2022-02-16T13:35:10.824481Z","shell.execute_reply":"2022-02-16T13:35:10.868731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#import necessary libraries\nimport numpy as np \nimport pandas as pd\nimport json\nfrom tqdm.notebook import tqdm\nimport matplotlib.pyplot as plt\nimport re\nimport gc\nimport seaborn as sns\n\nimport tensorflow as tf\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import CountVectorizer,TfidfVectorizer\n\nfrom tensorflow.keras.layers import LSTM, Bidirectional, GlobalMaxPooling1D, SpatialDropout1D, Dense, Dropout, Input, concatenate, Conv1D, Activation, Flatten","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:13.801330Z","iopub.execute_input":"2022-02-16T13:35:13.801854Z","iopub.status.idle":"2022-02-16T13:35:19.757461Z","shell.execute_reply.started":"2022-02-16T13:35:13.801815Z","shell.execute_reply":"2022-02-16T13:35:19.756689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/tensorflow2-question-answering/sample_submission.csv\")\nsample_submission.head(20)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:22.221175Z","iopub.execute_input":"2022-02-16T13:35:22.221984Z","iopub.status.idle":"2022-02-16T13:35:22.251621Z","shell.execute_reply.started":"2022-02-16T13:35:22.221935Z","shell.execute_reply":"2022-02-16T13:35:22.250877Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#path for data files\ntrain_path = '../input/tensorflow2-question-answering/simplified-nq-train.jsonl'\ntest_path = '../input/tensorflow2-question-answering/simplified-nq-test.jsonl'","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:25.068340Z","iopub.execute_input":"2022-02-16T13:35:25.068930Z","iopub.status.idle":"2022-02-16T13:35:25.073070Z","shell.execute_reply.started":"2022-02-16T13:35:25.068894Z","shell.execute_reply":"2022-02-16T13:35:25.072366Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"chunksize=2000","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:26.992042Z","iopub.execute_input":"2022-02-16T13:35:26.992348Z","iopub.status.idle":"2022-02-16T13:35:26.997129Z","shell.execute_reply.started":"2022-02-16T13:35:26.992315Z","shell.execute_reply":"2022-02-16T13:35:26.996302Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#read a sample of data files\ndef read_data(path, sample = True, chunksize = chunksize):\n    if sample == True:\n        df = []\n        with open(path, 'r') as reader:\n            for i in range(chunksize):\n                df.append(json.loads(reader.readline()))\n        df = pd.DataFrame(df)\n        print('Sample data have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n    else:\n        df = pd.read_json(path, orient = 'records', lines = True)\n        print('Sample data have {} rows and {} columns'.format(df.shape[0], df.shape[1]))\n        gc.collect()\n    return df\n\ntrain = read_data(train_path, sample = True)\ntest = read_data(test_path, sample = False)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:29.179538Z","iopub.execute_input":"2022-02-16T13:35:29.179832Z","iopub.status.idle":"2022-02-16T13:35:32.929850Z","shell.execute_reply.started":"2022-02-16T13:35:29.179800Z","shell.execute_reply":"2022-02-16T13:35:32.929061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploratory Data Analysis","metadata":{}},{"cell_type":"code","source":"df = read_data(train_path, sample = True)\nprint(df['question_text'][1])\nprint(df['annotations'][1])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:37.311490Z","iopub.execute_input":"2022-02-16T13:35:37.312366Z","iopub.status.idle":"2022-02-16T13:35:38.313086Z","shell.execute_reply.started":"2022-02-16T13:35:37.312315Z","shell.execute_reply":"2022-02-16T13:35:38.312317Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check the distribution of word count in sample 10000 documents\ndoc_text_words = df['document_text'].apply(lambda x: len(x.split(' ')))\nplt.figure(figsize=(12,6))\nsns.kdeplot(doc_text_words.values,shade=True, color=\"r\").set_title('Distribution of text word count of ' + str(chunksize) + ' documents')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:40.079112Z","iopub.execute_input":"2022-02-16T13:35:40.079842Z","iopub.status.idle":"2022-02-16T13:35:41.238282Z","shell.execute_reply.started":"2022-02-16T13:35:40.079804Z","shell.execute_reply":"2022-02-16T13:35:41.237591Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#long_answer_candidates\ndf.long_answer_candidates[0][:5]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:45.302537Z","iopub.execute_input":"2022-02-16T13:35:45.303100Z","iopub.status.idle":"2022-02-16T13:35:45.309489Z","shell.execute_reply.started":"2022-02-16T13:35:45.303061Z","shell.execute_reply":"2022-02-16T13:35:45.308756Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.annotations[1][:100]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:47.225362Z","iopub.execute_input":"2022-02-16T13:35:47.226105Z","iopub.status.idle":"2022-02-16T13:35:47.232261Z","shell.execute_reply.started":"2022-02-16T13:35:47.226067Z","shell.execute_reply":"2022-02-16T13:35:47.231299Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.question_text[1][:100]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:49.778528Z","iopub.execute_input":"2022-02-16T13:35:49.778788Z","iopub.status.idle":"2022-02-16T13:35:49.784972Z","shell.execute_reply.started":"2022-02-16T13:35:49.778757Z","shell.execute_reply":"2022-02-16T13:35:49.784166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.document_text[3][:100]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:51.979980Z","iopub.execute_input":"2022-02-16T13:35:51.980679Z","iopub.status.idle":"2022-02-16T13:35:51.986985Z","shell.execute_reply.started":"2022-02-16T13:35:51.980642Z","shell.execute_reply":"2022-02-16T13:35:51.986323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check long_answers_distribution\ndef preprocess_data():\n    df = read_data(train_path, sample = True)\n    df['yes_no'] = df.annotations.apply(lambda x: x[0]['yes_no_answer'])\n    df['long'] = df.annotations.apply(lambda x: [x[0]['long_answer']['start_token'], x[0]['long_answer']['end_token']])\n    df['short'] = df.annotations.apply(lambda x: x[0]['short_answers'])\n    return df\ndf = preprocess_data()\ndf","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:54.027803Z","iopub.execute_input":"2022-02-16T13:35:54.028520Z","iopub.status.idle":"2022-02-16T13:35:55.060175Z","shell.execute_reply.started":"2022-02-16T13:35:54.028482Z","shell.execute_reply":"2022-02-16T13:35:55.059491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['yes_no'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:35:58.419582Z","iopub.execute_input":"2022-02-16T13:35:58.420254Z","iopub.status.idle":"2022-02-16T13:35:58.428809Z","shell.execute_reply.started":"2022-02-16T13:35:58.420217Z","shell.execute_reply":"2022-02-16T13:35:58.428058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# filter the answers that exist\ndf_ans_exists = df.long.apply(lambda x: \"Answer Doesn't exist\" if x == -1 else \"Answer Exists\") == \"Answer Exists\"\ndf_ans_exists.count()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:00.799524Z","iopub.execute_input":"2022-02-16T13:36:00.800106Z","iopub.status.idle":"2022-02-16T13:36:00.811323Z","shell.execute_reply.started":"2022-02-16T13:36:00.800068Z","shell.execute_reply":"2022-02-16T13:36:00.810565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check if there are questions that do not have an answer\ndisplay(df.long.apply(lambda x: \"Answer Doesn't exist\" if x[0] == -1 else \"Answer Exists\").value_counts(normalize=True))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:03.239594Z","iopub.execute_input":"2022-02-16T13:36:03.240302Z","iopub.status.idle":"2022-02-16T13:36:03.251145Z","shell.execute_reply.started":"2022-02-16T13:36:03.240249Z","shell.execute_reply":"2022-02-16T13:36:03.250259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check the dstribution of Yes and No Answers\nyes_no_dist = df[df_ans_exists].yes_no.value_counts(normalize=True)\ndisplay(yes_no_dist)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:04.870661Z","iopub.execute_input":"2022-02-16T13:36:04.871120Z","iopub.status.idle":"2022-02-16T13:36:04.883924Z","shell.execute_reply.started":"2022-02-16T13:36:04.871082Z","shell.execute_reply":"2022-02-16T13:36:04.883215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#checking the distribution of short answers\nshort_dist = df[df_ans_exists].short.apply(lambda x: \"Short answer exists\" if len(x) > 0 else \"Short answer doesn't exist\").value_counts(normalize=True)\nplt.figure(figsize=(8,6))\nsns.barplot(x=short_dist.index,y=short_dist.values,hue=short_dist.index,palette = \"Reds\").set_title(\"Distribution of short answers in answerable questions\")","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:07.047977Z","iopub.execute_input":"2022-02-16T13:36:07.048370Z","iopub.status.idle":"2022-02-16T13:36:07.283376Z","shell.execute_reply.started":"2022-02-16T13:36:07.048338Z","shell.execute_reply":"2022-02-16T13:36:07.282702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_ans_dist = df[df_ans_exists].short.apply(len).value_counts(normalize=True)\nshort_ans_dist = pd.concat([short_ans_dist.loc[[0,1,],], pd.Series(short_ans_dist.loc[3:].sum(),index=['>=3'])])\nshort_ans_dist = short_ans_dist.rename(index={0: 'No Short answer',1:\"1 or 2 Short answers\",\">=3\":\"More than 2 short answers\"})\nplt.figure(figsize=(12,6))\nsns.barplot(x=short_ans_dist.index,y=short_ans_dist.values,hue=short_ans_dist.index,palette = \"Reds\").set_title(\"Distribution of Number of Short Answers in answerable questions\")","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:09.930696Z","iopub.execute_input":"2022-02-16T13:36:09.931609Z","iopub.status.idle":"2022-02-16T13:36:10.259543Z","shell.execute_reply.started":"2022-02-16T13:36:09.931562Z","shell.execute_reply":"2022-02-16T13:36:10.258884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check if there are missing data in the datasets\ndef check_missing_data(df):\n    missing_value=df.isna().sum().any()\n    if missing_value==True:\n        total = df.isnull().sum()\n        percent = (df.isnull().sum())/(df.isnull().count()*100)\n        output = pd.concat([total, percent], axis=1, keys=['Total', 'Percent'])\n        data_type = []\n\n        for col in df.columns:\n            dtype = str(train[col].dtype)\n            data_type.append(dtype)\n        output['Types'] = data_type\n        return(np.transpose(output))\n    else:\n        return(False)\n    \nprint(\"Missing data exists in train set: \",check_missing_data(train))\nprint(\"Missing data exists in test set: \",check_missing_data(test))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:13.362456Z","iopub.execute_input":"2022-02-16T13:36:13.362718Z","iopub.status.idle":"2022-02-16T13:36:13.377658Z","shell.execute_reply.started":"2022-02-16T13:36:13.362688Z","shell.execute_reply":"2022-02-16T13:36:13.376669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#check count values in each column\ndef count_values_in_column(data,feature):\n    total=data.loc[:,feature].value_counts(dropna=False)\n    percentage=round(data.loc[:,feature].value_counts(dropna=False,normalize=True)*100,2)\n    return pd.concat([total,percentage],axis=1,keys=['Total','Percentage'])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:16.528544Z","iopub.execute_input":"2022-02-16T13:36:16.529138Z","iopub.status.idle":"2022-02-16T13:36:16.534174Z","shell.execute_reply.started":"2022-02-16T13:36:16.529098Z","shell.execute_reply":"2022-02-16T13:36:16.533483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_values_in_column(df,'yes_no')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:19.442866Z","iopub.execute_input":"2022-02-16T13:36:19.443428Z","iopub.status.idle":"2022-02-16T13:36:19.457096Z","shell.execute_reply.started":"2022-02-16T13:36:19.443385Z","shell.execute_reply":"2022-02-16T13:36:19.456382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#count unique values in each column\ndef unique_values_in_column(data,feature):\n    unique_val=pd.Series(data.loc[:,feature].unique())\n    return pd.concat([unique_val],axis=1,keys=['Unique Values'])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:21.321113Z","iopub.execute_input":"2022-02-16T13:36:21.321906Z","iopub.status.idle":"2022-02-16T13:36:21.327403Z","shell.execute_reply.started":"2022-02-16T13:36:21.321847Z","shell.execute_reply":"2022-02-16T13:36:21.326634Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values_in_column(df,'document_text')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:23.938991Z","iopub.execute_input":"2022-02-16T13:36:23.939542Z","iopub.status.idle":"2022-02-16T13:36:24.241371Z","shell.execute_reply.started":"2022-02-16T13:36:23.939505Z","shell.execute_reply":"2022-02-16T13:36:24.240610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#find duplicated values in each column\ndef duplicated_values_data(data):\n    dup=[]\n    columns=data.columns\n    for i in data.columns:\n        dup.append(sum(data[i].duplicated()))\n    return pd.concat([pd.Series(columns),pd.Series(dup)],axis=1,keys=['Columns','Duplicate count'])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:26.580733Z","iopub.execute_input":"2022-02-16T13:36:26.581472Z","iopub.status.idle":"2022-02-16T13:36:26.588089Z","shell.execute_reply.started":"2022-02-16T13:36:26.581433Z","shell.execute_reply":"2022-02-16T13:36:26.585970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"duplicated_values_data(df)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:30.579140Z","iopub.execute_input":"2022-02-16T13:36:30.579579Z","iopub.status.idle":"2022-02-16T13:36:31.518578Z","shell.execute_reply.started":"2022-02-16T13:36:30.579543Z","shell.execute_reply":"2022-02-16T13:36:31.517859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df.describe()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:34.951676Z","iopub.execute_input":"2022-02-16T13:36:34.951934Z","iopub.status.idle":"2022-02-16T13:36:34.968641Z","shell.execute_reply.started":"2022-02-16T13:36:34.951903Z","shell.execute_reply":"2022-02-16T13:36:34.967959Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#remove hash texts in questions\ndef find_hash(text):\n    line=re.findall(r'(?<=#)\\w+',text)\n    return \" \".join(line)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:36.759684Z","iopub.execute_input":"2022-02-16T13:36:36.759959Z","iopub.status.idle":"2022-02-16T13:36:36.765248Z","shell.execute_reply.started":"2022-02-16T13:36:36.759927Z","shell.execute_reply":"2022-02-16T13:36:36.764081Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['hash']=df['question_text'].apply(lambda x: find_hash(x))\n","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:39.131749Z","iopub.execute_input":"2022-02-16T13:36:39.132038Z","iopub.status.idle":"2022-02-16T13:36:39.143951Z","shell.execute_reply.started":"2022-02-16T13:36:39.131985Z","shell.execute_reply":"2022-02-16T13:36:39.143146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['hash'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:41.319224Z","iopub.execute_input":"2022-02-16T13:36:41.319560Z","iopub.status.idle":"2022-02-16T13:36:41.327204Z","shell.execute_reply.started":"2022-02-16T13:36:41.319525Z","shell.execute_reply":"2022-02-16T13:36:41.326425Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unique_values_in_column(df,'hash').value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:43.259830Z","iopub.execute_input":"2022-02-16T13:36:43.260424Z","iopub.status.idle":"2022-02-16T13:36:43.269511Z","shell.execute_reply.started":"2022-02-16T13:36:43.260364Z","shell.execute_reply":"2022-02-16T13:36:43.268716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df['hash'] = df['hash'].fillna(df['document_text'])\ndf.loc[df['hash'].str.strip() == '', 'hash'] = df['document_text']","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:48.859352Z","iopub.execute_input":"2022-02-16T13:36:48.859889Z","iopub.status.idle":"2022-02-16T13:36:48.866960Z","shell.execute_reply.started":"2022-02-16T13:36:48.859844Z","shell.execute_reply":"2022-02-16T13:36:48.865995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:51.999918Z","iopub.execute_input":"2022-02-16T13:36:52.000818Z","iopub.status.idle":"2022-02-16T13:36:52.262792Z","shell.execute_reply.started":"2022-02-16T13:36:52.000775Z","shell.execute_reply":"2022-02-16T13:36:52.262052Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#remove punctuations\ndef find_punct(text):\n    line = re.findall(r'[!\"\\$%&\\'()*+,\\-.\\/:;=#@?\\[\\\\\\]^_`{|}~]*', text)\n    string=\"\".join(line)\n    return list(string)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:56.340854Z","iopub.execute_input":"2022-02-16T13:36:56.341558Z","iopub.status.idle":"2022-02-16T13:36:56.345562Z","shell.execute_reply.started":"2022-02-16T13:36:56.341517Z","shell.execute_reply":"2022-02-16T13:36:56.344830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df['punctuation']=df['question_text'].apply(lambda x : find_punct(x))\ndf.loc[df['hash'].str.strip() == '', 'hash'] = df['question_text']","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:36:58.538601Z","iopub.execute_input":"2022-02-16T13:36:58.539406Z","iopub.status.idle":"2022-02-16T13:36:58.581272Z","shell.execute_reply.started":"2022-02-16T13:36:58.539354Z","shell.execute_reply":"2022-02-16T13:36:58.580613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:00.519278Z","iopub.execute_input":"2022-02-16T13:37:00.519764Z","iopub.status.idle":"2022-02-16T13:37:00.766431Z","shell.execute_reply.started":"2022-02-16T13:37:00.519726Z","shell.execute_reply":"2022-02-16T13:37:00.765736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"np.unique(df['punctuation'])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:04.868098Z","iopub.execute_input":"2022-02-16T13:37:04.868723Z","iopub.status.idle":"2022-02-16T13:37:04.875891Z","shell.execute_reply.started":"2022-02-16T13:37:04.868682Z","shell.execute_reply":"2022-02-16T13:37:04.874876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define training parameters \nnum_train_ques = 2000\nnum_val_ques = 2050\nsample_rate = 15","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:07.389841Z","iopub.execute_input":"2022-02-16T13:37:07.390105Z","iopub.status.idle":"2022-02-16T13:37:07.394506Z","shell.execute_reply.started":"2022-02-16T13:37:07.390076Z","shell.execute_reply":"2022-02-16T13:37:07.393840Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define model parameters\nepochs = 40\nbatch_size = 64\nclass_weights = {0: 0.5, 1: 5.}","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:09.979054Z","iopub.execute_input":"2022-02-16T13:37:09.979768Z","iopub.status.idle":"2022-02-16T13:37:09.985037Z","shell.execute_reply.started":"2022-02-16T13:37:09.979728Z","shell.execute_reply":"2022-02-16T13:37:09.984079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Part 1- identifying correct long answer to a question**","metadata":{}},{"cell_type":"code","source":"def get_question_and_document(line):\n    question = line['question_text']\n    text = line['document_text'].split(' ')\n    annotations = line['annotations'][0]    \n    return question, text, annotations\n                \ndef get_long_candidate(i, annotations, candidate):\n    # check if this candidate is the correct long answer\n    if i == annotations['long_answer']['candidate_index']:\n        label = True\n    else:\n        label = False\n\n    # get place where long answer starts and ends in the document text\n    long_start = candidate['start_token']\n    long_end = candidate['end_token']    \n    return label, long_start, long_end\n\ndef form_data_row(question, label, text, long_start, long_end):\n    row = {\n        'question': question,\n        'long_answer': ' '.join(text[long_start:long_end]),\n        'is_long_answer': label,\n    }    \n    return row","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:12.508207Z","iopub.execute_input":"2022-02-16T13:37:12.508661Z","iopub.status.idle":"2022-02-16T13:37:12.515199Z","shell.execute_reply.started":"2022-02-16T13:37:12.508624Z","shell.execute_reply":"2022-02-16T13:37:12.514402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(file_path, questions_start, questions_end):\n    rows = []\n    \n    with open(file_path) as file:\n        for i in tqdm(range(questions_start, questions_end)):\n            line = file.readline()\n            line = json.loads(line)\n            question, text, annotations = get_question_and_document(line)\n\n            for i, candidate in enumerate(line['long_answer_candidates']):\n                label, long_start, long_end = get_long_candidate(i, annotations, candidate)\n\n                if label == True or (i % sample_rate == 0):\n                    rows.append(\n                        form_data_row(question, label, text, long_start, long_end)\n                    )        \n    return pd.DataFrame(rows)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:15.231675Z","iopub.execute_input":"2022-02-16T13:37:15.232215Z","iopub.status.idle":"2022-02-16T13:37:15.238870Z","shell.execute_reply.started":"2022-02-16T13:37:15.232173Z","shell.execute_reply":"2022-02-16T13:37:15.238050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = load_data(train_path, 0, num_train_ques)\ntest_df = load_data(train_path, num_train_ques, num_val_ques)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:18.088496Z","iopub.execute_input":"2022-02-16T13:37:18.088751Z","iopub.status.idle":"2022-02-16T13:37:20.025819Z","shell.execute_reply.started":"2022-02-16T13:37:18.088722Z","shell.execute_reply":"2022-02-16T13:37:20.025025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:24.049364Z","iopub.execute_input":"2022-02-16T13:37:24.049964Z","iopub.status.idle":"2022-02-16T13:37:24.062468Z","shell.execute_reply.started":"2022-02-16T13:37:24.049926Z","shell.execute_reply":"2022-02-16T13:37:24.061566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:27.380328Z","iopub.execute_input":"2022-02-16T13:37:27.380870Z","iopub.status.idle":"2022-02-16T13:37:27.390493Z","shell.execute_reply.started":"2022-02-16T13:37:27.380828Z","shell.execute_reply":"2022-02-16T13:37:27.389731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(10)['long_answer']","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:29.551044Z","iopub.execute_input":"2022-02-16T13:37:29.551603Z","iopub.status.idle":"2022-02-16T13:37:29.558939Z","shell.execute_reply.started":"2022-02-16T13:37:29.551564Z","shell.execute_reply":"2022-02-16T13:37:29.558133Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Pre-processing texts**","metadata":{"execution":{"iopub.status.busy":"2022-01-07T13:00:09.060209Z","iopub.execute_input":"2022-01-07T13:00:09.060503Z","iopub.status.idle":"2022-01-07T13:00:09.065406Z","shell.execute_reply.started":"2022-01-07T13:00:09.060471Z","shell.execute_reply":"2022-01-07T13:00:09.064588Z"}}},{"cell_type":"code","source":"#cleaning texts by removing stopwords \ndef remove_stopwords(sentence):\n    words = sentence.split()\n    words = [word for word in words if word not in stopwords.words('english')]\n    return ' '.join(words)\n\n#removing html tags \ndef remove_html(sentence):\n    html = re.compile(r'<.*?>` `` ')\n    return html.sub(r'', sentence)\n\n#returns the pre-processed dataframe for long answers and questions\ndef preprocessed_df(df):\n    df['long_answer'] = df['long_answer'].apply(lambda x : remove_stopwords(x))\n    df['long_answer'] = df['long_answer'].apply(lambda x : remove_html(x))\n\n    df['question'] = df['question'].apply(lambda x : remove_stopwords(x))\n    df['question'] = df['question'].apply(lambda x : remove_html(x))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:31.760640Z","iopub.execute_input":"2022-02-16T13:37:31.761195Z","iopub.status.idle":"2022-02-16T13:37:31.768440Z","shell.execute_reply.started":"2022-02-16T13:37:31.761156Z","shell.execute_reply":"2022-02-16T13:37:31.767735Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Count vectorizer for N grams\n\ndef ngrams_top(corpus,ngram_range,n=None):\n    \"\"\"\n    List the top n words in a vocabulary according to occurrence in a text corpus.\n    \"\"\"\n    vec = CountVectorizer(stop_words = 'english',ngram_range=ngram_range).fit(corpus)\n    bag_of_words = vec.transform(corpus)\n    sum_words = bag_of_words.sum(axis=0) \n    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]\n    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)\n    total_list=words_freq[:n]\n    df=pd.DataFrame(total_list,columns=['text','count'])\n    return df","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:34.207573Z","iopub.execute_input":"2022-02-16T13:37:34.207878Z","iopub.status.idle":"2022-02-16T13:37:34.218810Z","shell.execute_reply.started":"2022-02-16T13:37:34.207844Z","shell.execute_reply":"2022-02-16T13:37:34.218093Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ngrams_top(df['question_text'],(3,3),n=10)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:36.451262Z","iopub.execute_input":"2022-02-16T13:37:36.451750Z","iopub.status.idle":"2022-02-16T13:37:36.542175Z","shell.execute_reply.started":"2022-02-16T13:37:36.451710Z","shell.execute_reply":"2022-02-16T13:37:36.541404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = preprocessed_df(train_df)\ntest_df = preprocessed_df(test_df)\ntrain_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:37:38.701309Z","iopub.execute_input":"2022-02-16T13:37:38.703305Z","iopub.status.idle":"2022-02-16T13:42:01.179264Z","shell.execute_reply.started":"2022-02-16T13:37:38.703256Z","shell.execute_reply":"2022-02-16T13:42:01.178583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df['long_answer']","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:08.151127Z","iopub.execute_input":"2022-02-16T13:42:08.151576Z","iopub.status.idle":"2022-02-16T13:42:08.158559Z","shell.execute_reply.started":"2022-02-16T13:42:08.151538Z","shell.execute_reply":"2022-02-16T13:42:08.157556Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tokenizing the texts","metadata":{}},{"cell_type":"code","source":"# define tokenization parameters\nfilters = '!\"''#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n'\nlower_case = True\nmax_len = 500 #max lenght of a sentence input in to the model","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:11.010134Z","iopub.execute_input":"2022-02-16T13:42:11.010688Z","iopub.status.idle":"2022-02-16T13:42:11.014537Z","shell.execute_reply.started":"2022-02-16T13:42:11.010647Z","shell.execute_reply":"2022-02-16T13:42:11.013578Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#assigning a numeric index to each unique word in the dataset so that an array of integers\ndef define_tokenizer(series):\n    sentences = pd.concat(series)    \n    tokenizer = tf.keras.preprocessing.text.Tokenizer(lower=lower_case,filters=filters  )\n    tokenizer.fit_on_texts(sentences)\n    return tokenizer\n\n#encoding, rest with 0 if max_len>no.of words in sentence \ndef encode(sentences, tokenizer):\n    encoded_sentences = tokenizer.texts_to_sequences(sentences)\n    encoded_sentences = tf.keras.preprocessing.sequence.pad_sequences(encoded_sentences,\n                                                                      maxlen=max_len, padding='post')\n    return encoded_sentences","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:13.256299Z","iopub.execute_input":"2022-02-16T13:42:13.256772Z","iopub.status.idle":"2022-02-16T13:42:13.262952Z","shell.execute_reply.started":"2022-02-16T13:42:13.256731Z","shell.execute_reply":"2022-02-16T13:42:13.262277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = define_tokenizer([train_df.long_answer,train_df.question,\n                              test_df.long_answer,test_df.question])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:20.029737Z","iopub.execute_input":"2022-02-16T13:42:20.029997Z","iopub.status.idle":"2022-02-16T13:42:21.544098Z","shell.execute_reply.started":"2022-02-16T13:42:20.029967Z","shell.execute_reply":"2022-02-16T13:42:21.543371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:22.568672Z","iopub.execute_input":"2022-02-16T13:42:22.569322Z","iopub.status.idle":"2022-02-16T13:42:22.576097Z","shell.execute_reply.started":"2022-02-16T13:42:22.569236Z","shell.execute_reply":"2022-02-16T13:42:22.574357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the training data word index\nword_index = tokenizer.word_index\nprint('Number of Unique Tokens: %d' % len(word_index))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:24.560035Z","iopub.execute_input":"2022-02-16T13:42:24.560717Z","iopub.status.idle":"2022-02-16T13:42:24.565182Z","shell.execute_reply.started":"2022-02-16T13:42:24.560678Z","shell.execute_reply":"2022-02-16T13:42:24.564238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"list(enumerate(word_index.items()))[:50]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:26.840253Z","iopub.execute_input":"2022-02-16T13:42:26.840958Z","iopub.status.idle":"2022-02-16T13:42:26.878730Z","shell.execute_reply.started":"2022-02-16T13:42:26.840908Z","shell.execute_reply":"2022-02-16T13:42:26.877936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer.word_index['handle']","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:29.880569Z","iopub.execute_input":"2022-02-16T13:42:29.881314Z","iopub.status.idle":"2022-02-16T13:42:29.887184Z","shell.execute_reply.started":"2022-02-16T13:42:29.881263Z","shell.execute_reply":"2022-02-16T13:42:29.886385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer.word_index['pandemic']","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:31.600147Z","iopub.execute_input":"2022-02-16T13:42:31.600694Z","iopub.status.idle":"2022-02-16T13:42:31.607549Z","shell.execute_reply.started":"2022-02-16T13:42:31.600656Z","shell.execute_reply":"2022-02-16T13:42:31.606716Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_long_answers = encode(train_df['long_answer'].values, tokenizer)\ntrain_questions = encode(train_df['question'].values, tokenizer)\n\ntest_long_answers = encode(test_df['long_answer'].values, tokenizer)\ntest_questions = encode(test_df['question'].values, tokenizer)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:33.720149Z","iopub.execute_input":"2022-02-16T13:42:33.720712Z","iopub.status.idle":"2022-02-16T13:42:35.332403Z","shell.execute_reply.started":"2022-02-16T13:42:33.720672Z","shell.execute_reply":"2022-02-16T13:42:35.331646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_long_answers","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:42.597379Z","iopub.execute_input":"2022-02-16T13:42:42.598129Z","iopub.status.idle":"2022-02-16T13:42:42.607274Z","shell.execute_reply.started":"2022-02-16T13:42:42.598091Z","shell.execute_reply":"2022-02-16T13:42:42.606553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_long_answers[0]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:44.980784Z","iopub.execute_input":"2022-02-16T13:42:44.981066Z","iopub.status.idle":"2022-02-16T13:42:44.989608Z","shell.execute_reply.started":"2022-02-16T13:42:44.981035Z","shell.execute_reply":"2022-02-16T13:42:44.988868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_questions","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:51.081193Z","iopub.execute_input":"2022-02-16T13:42:51.081450Z","iopub.status.idle":"2022-02-16T13:42:51.086972Z","shell.execute_reply.started":"2022-02-16T13:42:51.081421Z","shell.execute_reply":"2022-02-16T13:42:51.086323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels = train_df.is_long_answer.astype(int).values\ntest_labels = test_df.is_long_answer.astype(int).values","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:53.120307Z","iopub.execute_input":"2022-02-16T13:42:53.120859Z","iopub.status.idle":"2022-02-16T13:42:53.125424Z","shell.execute_reply.started":"2022-02-16T13:42:53.120816Z","shell.execute_reply":"2022-02-16T13:42:53.124701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_labels","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:55.260390Z","iopub.execute_input":"2022-02-16T13:42:55.260930Z","iopub.status.idle":"2022-02-16T13:42:55.266402Z","shell.execute_reply.started":"2022-02-16T13:42:55.260893Z","shell.execute_reply":"2022-02-16T13:42:55.265708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_labels","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:42:57.289798Z","iopub.execute_input":"2022-02-16T13:42:57.290347Z","iopub.status.idle":"2022-02-16T13:42:57.297327Z","shell.execute_reply.started":"2022-02-16T13:42:57.290306Z","shell.execute_reply":"2022-02-16T13:42:57.296638Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Define embedding layer","metadata":{}},{"cell_type":"markdown","source":" Loading the pre-trained embedding - using Glove","metadata":{}},{"cell_type":"code","source":"embed_size = 200\nembedding_dict = {}\n\nwith open('../input/glove6b/glove.6B.' + str(embed_size) + 'd.txt','r') as f:\n    for line in f:\n        values = line.split()\n        word = values[0]\n        vectors = np.asarray(values[1:],'float32')\n        embedding_dict[word] = vectors        \nf.close()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:43:00.900110Z","iopub.execute_input":"2022-02-16T13:43:00.900634Z","iopub.status.idle":"2022-02-16T13:43:32.171707Z","shell.execute_reply.started":"2022-02-16T13:43:00.900597Z","shell.execute_reply":"2022-02-16T13:43:32.170988Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_words = len(word_index) + 1\n\n#initializing embedding matrix\nembedding_matrix = np.zeros((num_words, embed_size))\nprint(\"num_words:\", num_words)\nprint(\"embedding_matrix:\", embedding_matrix)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:43:40.280401Z","iopub.execute_input":"2022-02-16T13:43:40.280940Z","iopub.status.idle":"2022-02-16T13:43:40.287989Z","shell.execute_reply.started":"2022-02-16T13:43:40.280899Z","shell.execute_reply":"2022-02-16T13:43:40.287066Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for word, i in word_index.items():\n    if i > num_words:\n        continue\n    \n    emb_vec = embedding_dict.get(word)\n    \n    if emb_vec is not None:\n        embedding_matrix[i] = emb_vec","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:43:44.279348Z","iopub.execute_input":"2022-02-16T13:43:44.280038Z","iopub.status.idle":"2022-02-16T13:43:44.460304Z","shell.execute_reply.started":"2022-02-16T13:43:44.279978Z","shell.execute_reply":"2022-02-16T13:43:44.459553Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Defining model¶\n","metadata":{}},{"cell_type":"code","source":"#initialize embedding layer\nembedding_layer = tf.keras.layers.Embedding(\n    len(word_index) + 1,\n    embed_size,\n    embeddings_initializer = tf.keras.initializers.Constant(embedding_matrix),\n    trainable = False\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:43:53.811989Z","iopub.execute_input":"2022-02-16T13:43:53.812688Z","iopub.status.idle":"2022-02-16T13:43:53.840995Z","shell.execute_reply.started":"2022-02-16T13:43:53.812648Z","shell.execute_reply":"2022-02-16T13:43:53.840264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# question encoding-encodes the question\nquestion_input = Input(shape=(None,))\nquestion_x = embedding_layer(question_input)\nquestion_x = SpatialDropout1D(0.2)(question_x)\nquestion_x = Bidirectional(LSTM(100, return_sequences=True))(question_x)\nquestion_x = GlobalMaxPooling1D()(question_x) # outputs an encoded array representing the question\n\n# answer encoding-encodes the answer\nanswer_input = Input(shape=(None,))\nanswer_x = embedding_layer(answer_input)\nanswer_x = SpatialDropout1D(0.2)(answer_x)\nanswer_x = Bidirectional(LSTM(150, return_sequences=True))(answer_x)\nanswer_x = GlobalMaxPooling1D()(answer_x) #outputs an encoded array representing the answer\n\n# classification\ncombined_x = concatenate([question_x, answer_x])\ncombined_x = Dense(300, activation='relu')(combined_x)\ncombined_x = Dropout(0.5)(combined_x)\ncombined_x = Dense(300, activation='relu')(combined_x)\ncombined_x = Dropout(0.5)(combined_x)\nmodel_output = Dense(1, activation='sigmoid')(combined_x) # probability how close the potential answer is to the true answer to the question.\n\n# defining model by combining above three parts\nmodel = tf.keras.models.Model(inputs=[answer_input, question_input], outputs=model_output)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:43:58.260445Z","iopub.execute_input":"2022-02-16T13:43:58.260912Z","iopub.status.idle":"2022-02-16T13:44:01.711289Z","shell.execute_reply.started":"2022-02-16T13:43:58.260872Z","shell.execute_reply":"2022-02-16T13:44:01.710537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:44:05.731540Z","iopub.execute_input":"2022-02-16T13:44:05.731806Z","iopub.status.idle":"2022-02-16T13:44:05.746856Z","shell.execute_reply.started":"2022-02-16T13:44:05.731773Z","shell.execute_reply":"2022-02-16T13:44:05.746136Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.compile(\n    loss='binary_crossentropy', \n    optimizer='adam',\n    metrics=['BinaryAccuracy', 'Recall', 'Precision'])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:44:08.401164Z","iopub.execute_input":"2022-02-16T13:44:08.401950Z","iopub.status.idle":"2022-02-16T13:44:08.416766Z","shell.execute_reply.started":"2022-02-16T13:44:08.401892Z","shell.execute_reply":"2022-02-16T13:44:08.415908Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define callbacks - to avoid plateauing & achieve early stopping\ncallbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=2, verbose=1),\n    tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1),]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:44:10.920121Z","iopub.execute_input":"2022-02-16T13:44:10.920669Z","iopub.status.idle":"2022-02-16T13:44:10.924883Z","shell.execute_reply.started":"2022-02-16T13:44:10.920631Z","shell.execute_reply":"2022-02-16T13:44:10.923947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define model parameters\nepochs = 30\nbatch_size = 128\nclass_weights = {0: 0.5, 1: 5.}","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:44:12.948324Z","iopub.execute_input":"2022-02-16T13:44:12.950192Z","iopub.status.idle":"2022-02-16T13:44:12.954176Z","shell.execute_reply.started":"2022-02-16T13:44:12.950142Z","shell.execute_reply":"2022-02-16T13:44:12.953424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(\n    x = [train_long_answers, train_questions], \n    y = train_labels,\n    validation_data = (\n        [test_long_answers, test_questions], test_labels),\n    epochs = epochs,\n    callbacks = callbacks,\n    class_weight = class_weights,\n    batch_size = batch_size,\n    shuffle = True\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T13:44:15.147874Z","iopub.execute_input":"2022-02-16T13:44:15.148441Z","iopub.status.idle":"2022-02-16T13:59:42.251064Z","shell.execute_reply.started":"2022-02-16T13:44:15.148401Z","shell.execute_reply":"2022-02-16T13:59:42.250265Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#save model\nmodel.save('long_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:00:03.862580Z","iopub.execute_input":"2022-02-16T14:00:03.862854Z","iopub.status.idle":"2022-02-16T14:00:16.529822Z","shell.execute_reply.started":"2022-02-16T14:00:03.862822Z","shell.execute_reply":"2022-02-16T14:00:16.529085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model evaluation","metadata":{}},{"cell_type":"markdown","source":"identifying if correct long answers given to a question","metadata":{}},{"cell_type":"code","source":"#Evaluate checking training & validation loss\nfig, ax = plt.subplots(1, 2, figsize=(15, 5))\n\nax[0].set_title('Training Loss')\nax[0].plot(history.history['loss'],marker = '.',mec = 'r', mfc = 'r')\n\nax[1].set_title('Validation Loss')\nax[1].plot(history.history['val_loss'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:00:18.590549Z","iopub.execute_input":"2022-02-16T14:00:18.590828Z","iopub.status.idle":"2022-02-16T14:00:18.963075Z","shell.execute_reply.started":"2022-02-16T14:00:18.590794Z","shell.execute_reply":"2022-02-16T14:00:18.962413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Evaluate checking accuracy, recall and precision\nfig, ax = plt.subplots(3, 2, figsize=(15, 10))\n\nax[0,0].set_title('Training Accuracy')\nax[0,0].plot(history.history['binary_accuracy'],marker = '.',mec = 'r', mfc = 'r')\n\nax[0,1].set_title('Validation Accuracy')\nax[0,1].plot(history.history['val_binary_accuracy'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")\n\nax[1,0].set_title('Training Recall')\nax[1,0].plot(history.history['recall'],marker = '.',mec = 'r', mfc = 'r')\n\nax[1,1].set_title('Validation Recall')\nax[1,1].plot(history.history['val_recall'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")\n\nax[2,0].set_title('Training Precision')\nax[2,0].plot(history.history['precision'],marker = '.',mec = 'r', mfc = 'r')\n\nax[2,1].set_title('Validation Precision')\nax[2,1].plot(history.history['val_precision'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:00:25.723746Z","iopub.execute_input":"2022-02-16T14:00:25.724059Z","iopub.status.idle":"2022-02-16T14:00:26.525352Z","shell.execute_reply.started":"2022-02-16T14:00:25.724002Z","shell.execute_reply":"2022-02-16T14:00:26.524640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Evaluate checking recall\n\nrecall = history.history['recall'][-1]\nprecision = history.history['precision'][-1]\n\nprint('Train F1 score: {0:.4f}'.format(\n    2 * (precision * recall) / (precision + recall)\n))\n\nrecall = history.history['val_recall'][-1]\nprecision = history.history['val_precision'][-1]\n\nprint('Validation F1 score: {0:.4f}'.format(2 * (precision * recall) / (precision + recall)))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:00:34.285560Z","iopub.execute_input":"2022-02-16T14:00:34.285813Z","iopub.status.idle":"2022-02-16T14:00:34.294551Z","shell.execute_reply.started":"2022-02-16T14:00:34.285783Z","shell.execute_reply":"2022-02-16T14:00:34.292368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = model.predict([np.expand_dims(test_labels, axis=0), np.expand_dims(test_labels, axis=0)])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:05.193679Z","iopub.execute_input":"2022-02-16T14:01:05.193963Z","iopub.status.idle":"2022-02-16T14:01:06.307319Z","shell.execute_reply.started":"2022-02-16T14:01:05.193932Z","shell.execute_reply":"2022-02-16T14:01:06.306546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:07.855415Z","iopub.execute_input":"2022-02-16T14:01:07.855688Z","iopub.status.idle":"2022-02-16T14:01:07.863415Z","shell.execute_reply.started":"2022-02-16T14:01:07.855657Z","shell.execute_reply":"2022-02-16T14:01:07.860135Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#evaluating with live inputs\ndef test_question(question, positive, negative):\n    sentences = [question, positive, negative]\n    \n    for i in range(3):\n        sentences[i] = remove_stopwords(sentences[i])\n        sentences[i] = remove_html(sentences[i])\n    \n    sentences = encode(sentences, tokenizer)\n    \n    predictions = model.predict([np.expand_dims(sentences[1], axis=0), np.expand_dims(sentences[0], axis=0)])\n\n    print('Positive: {0:.2f}'.format(predictions[0][0]))\n\n    predictions = model.predict(\n        [np.expand_dims(sentences[2], axis=0), np.expand_dims(sentences[0], axis=0)])\n\n    print('Negative: {0:.2f}'.format(predictions[0][0]))\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:09.925007Z","iopub.execute_input":"2022-02-16T14:01:09.925633Z","iopub.status.idle":"2022-02-16T14:01:09.935722Z","shell.execute_reply.started":"2022-02-16T14:01:09.925590Z","shell.execute_reply":"2022-02-16T14:01:09.934709Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"question = 'who invented a portable handset '\n\npositive='<P> Martin Cooper invented a portable handset in 1973, when he was a project manager at Motorola. It was almost three decades after the idea of cellular communications was introduced by Bell Laboratories. </P>'\nnegative = '<P> Email marketing has evolved rapidly alongside the technological growth of the 21st century . Prior to this growth , when emails were novelties to the majority of customers , email marketing was not as effective . In 1978 , Gary Thuerk of Digital Equipment Corporation ( DEC ) sent out the first mass email to approximately 400 potential clients via the Advanced Research Projects Agency Network ( ARPANET ) . This email resulted in $13 million worth of sales in DEC products , and highlighted the potential of marketing through mass emails . However , as email marketing developed as an effective means of direct communication , users began blocking out content from emails with filters and blocking programs . In order to effectively communicate a message through email , marketers had to develop a way of pushing content through to the end user , without being cut out by automatic filters and spam removing software . This resulted in the birth of triggered marketing emails , which are sent to specific users based on their tracked online browsing patterns . </P>'","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:16.307711Z","iopub.execute_input":"2022-02-16T14:01:16.308471Z","iopub.status.idle":"2022-02-16T14:01:16.312586Z","shell.execute_reply.started":"2022-02-16T14:01:16.308428Z","shell.execute_reply":"2022-02-16T14:01:16.311851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_question(question, positive, negative)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:20.624752Z","iopub.execute_input":"2022-02-16T14:01:20.625606Z","iopub.status.idle":"2022-02-16T14:01:21.999503Z","shell.execute_reply.started":"2022-02-16T14:01:20.625565Z","shell.execute_reply":"2022-02-16T14:01:21.998750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"question = 'who is the south african high commissioner in london'\n\npositive = \"<P> Tracy McConnell , better known as `` The Mother '' , is the title character from the CBS television sitcom How I Met Your Mother . The show , narrated by Future Ted , tells the story of how Ted Mosby met The Mother . Tracy McConnell appears in 8 episodes from `` Lucky Penny '' to `` The Time Travelers '' as an unseen character ; she was first seen fully in `` Something New '' and was promoted to a main character in season 9 . The Mother is played by Cristin Milioti . </P>\"\n\nnegative = \"<P> In `` Bass Player Wanted '' , the Mother picks up a hitchhiking Marshall , carrying his son Marvin , on her way to Farhampton Inn . On their way , it is revealed that the Mother is a bass player in the band , that is scheduled to play at the wedding reception . But the band 's leader , Darren , forced her to quit . The Mother ultimately decides to confront Darren and retake the band . She ends up alone at the bar , and while practicing a speech to give Darren , Darren walks up to her furious the groom 's best man punched him for `` no reason . '' Amused by this , the Mother laughs , and Darren quits the band in anger . </P>\"","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:27.827766Z","iopub.execute_input":"2022-02-16T14:01:27.828353Z","iopub.status.idle":"2022-02-16T14:01:27.833438Z","shell.execute_reply.started":"2022-02-16T14:01:27.828311Z","shell.execute_reply":"2022-02-16T14:01:27.832434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_question(question, positive, negative)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:31.172087Z","iopub.execute_input":"2022-02-16T14:01:31.172761Z","iopub.status.idle":"2022-02-16T14:01:31.331222Z","shell.execute_reply.started":"2022-02-16T14:01:31.172724Z","shell.execute_reply":"2022-02-16T14:01:31.330404Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Part 2- extracting short answer from a long answer**","metadata":{}},{"cell_type":"code","source":"#filter records where short answers exists\ndef get_short_answer(annotations, long_start, long_end):\n    if len(annotations['short_answers']) > 0:\n        short_start = annotations['short_answers'][0]['start_token']\n        short_end = annotations['short_answers'][0]['end_token']        \n        short_start = short_start - long_start\n        short_end = short_end - long_start        \n        return short_start, short_end\n    else:\n        return 0, 0\n    \ndef form_short_data_row(question, text, long_start, long_end, short_start, short_end):\n    long_answer = ' '.join(text[long_start:long_end])\n    short_answer = ' '.join(long_answer.split(' ')[short_start:short_end])\n    row = {\n        'question': question,\n        'long_answer': long_answer,\n        'short_answer': short_answer,\n        'short_start': short_start,\n        'short_end': short_end\n    }    \n    return row","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:44.524785Z","iopub.execute_input":"2022-02-16T14:01:44.525075Z","iopub.status.idle":"2022-02-16T14:01:44.531906Z","shell.execute_reply.started":"2022-02-16T14:01:44.525043Z","shell.execute_reply":"2022-02-16T14:01:44.531235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#loading short answers\ndef load_short_data(file_path, questions_start, questions_end):\n    rows = []    \n    with open(file_path) as file:\n\n        for i in tqdm(range(questions_start, questions_end)):\n            line = file.readline()\n            line = json.loads(line)\n            question, text, annotations = get_question_and_document(line)\n\n            for i, candidate in enumerate(line['long_answer_candidates']):\n                label, long_start, long_end = get_long_candidate(i, annotations, candidate)\n\n                if label == True:\n                    short_start, short_end = get_short_answer(annotations, long_start, long_end)\n                    \n                    rows.append(\n                        form_short_data_row(question, text, long_start, long_end, short_start, short_end)\n                    )\n        \n    return pd.DataFrame(rows)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:48.213668Z","iopub.execute_input":"2022-02-16T14:01:48.214201Z","iopub.status.idle":"2022-02-16T14:01:48.220659Z","shell.execute_reply.started":"2022-02-16T14:01:48.214161Z","shell.execute_reply":"2022-02-16T14:01:48.219982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_short_df = load_short_data(train_path, 0, num_train_ques)\ntest_short_df = load_short_data(train_path, num_train_ques, num_val_ques)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:51.864914Z","iopub.execute_input":"2022-02-16T14:01:51.865675Z","iopub.status.idle":"2022-02-16T14:01:54.395433Z","shell.execute_reply.started":"2022-02-16T14:01:51.865630Z","shell.execute_reply":"2022-02-16T14:01:54.394667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_short_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:01:57.594710Z","iopub.execute_input":"2022-02-16T14:01:57.594988Z","iopub.status.idle":"2022-02-16T14:01:57.607731Z","shell.execute_reply.started":"2022-02-16T14:01:57.594955Z","shell.execute_reply":"2022-02-16T14:01:57.606942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count_values_in_column(train_short_df,'short_answer')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:02:00.503497Z","iopub.execute_input":"2022-02-16T14:02:00.504075Z","iopub.status.idle":"2022-02-16T14:02:00.521601Z","shell.execute_reply.started":"2022-02-16T14:02:00.504037Z","shell.execute_reply":"2022-02-16T14:02:00.520746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_long_answers = encode(train_short_df['long_answer'].values, tokenizer)\ntrain_questions = encode(train_short_df['question'].values, tokenizer)\n\ntest_long_answers = encode(test_short_df['long_answer'].values, tokenizer)\ntest_questions = encode(test_short_df['question'].values, tokenizer)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:02:04.263370Z","iopub.execute_input":"2022-02-16T14:02:04.264212Z","iopub.status.idle":"2022-02-16T14:02:04.588380Z","shell.execute_reply.started":"2022-02-16T14:02:04.264162Z","shell.execute_reply":"2022-02-16T14:02:04.587539Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#define 2 arrays for the start index and another for the end index\ndef form_short_labels(df, sentence_length):\n    start_labels = np.zeros((len(df), sentence_length))\n    end_labels = np.zeros((len(df), sentence_length))\n\n    #get the token indexes from short_start and short_end columns and assign it to new arrays. encoding with 1\n    for i in range(len(df)):\n        start = df.loc[i].short_start\n        end = df.loc[i].short_end\n\n        if start < 500 and end < 500:\n            start_labels[i, start] = 1\n            end_labels[i, end] = 1\n        else:\n            continue\n    \n    return start_labels, end_labels\n\ntrain_start_labels, train_end_labels = form_short_labels(train_short_df, max_len)\ntest_start_labels, test_end_labels = form_short_labels(test_short_df, max_len)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:02:14.254378Z","iopub.execute_input":"2022-02-16T14:02:14.254944Z","iopub.status.idle":"2022-02-16T14:02:14.501305Z","shell.execute_reply.started":"2022-02-16T14:02:14.254905Z","shell.execute_reply":"2022-02-16T14:02:14.500530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(train_short_df.loc[10].question)\n\nprint(train_short_df.loc[10].long_answer)\nprint(train_short_df.loc[10].short_answer)\n\nprint('Start index: {0}'.format(train_start_labels[10]))\nprint('End index: {0}'.format(train_end_labels[10]))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:02:19.043771Z","iopub.execute_input":"2022-02-16T14:02:19.044052Z","iopub.status.idle":"2022-02-16T14:02:19.060385Z","shell.execute_reply.started":"2022-02-16T14:02:19.044002Z","shell.execute_reply":"2022-02-16T14:02:19.059418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Defining the short model","metadata":{}},{"cell_type":"code","source":"# short answer model parameters\nshort_epochs = 100\nshort_batch_size = 64\nembed_size_short=200","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:03:12.885053Z","iopub.execute_input":"2022-02-16T14:03:12.885599Z","iopub.status.idle":"2022-02-16T14:03:12.888993Z","shell.execute_reply.started":"2022-02-16T14:03:12.885558Z","shell.execute_reply":"2022-02-16T14:03:12.888345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load from file\nembedding_dict = {}\n\nwith open('../input/glove6b/glove.6B.200' + 'd.txt','r') as f:\n    for line in f:\n        values = line.split()\n        word = values[0]\n        vectors = np.asarray(values[1:],'float32')\n        embedding_dict[word] = vectors\n        \nf.close()\n\n# write to matrix\nnum_words = len(tokenizer.word_index) + 1\nembedding_matrix = np.zeros((num_words, embed_size_short))\n\nfor word, i in tokenizer.word_index.items():\n    if i > num_words:\n        continue\n    \n    emb_vec = embedding_dict.get(word)\n    \n    if emb_vec is not None:\n        embedding_matrix[i] = emb_vec\n        \n# load as tensorflow embedding\n#define embedding layer for the short model\nembedding_layer2 = tf.keras.layers.Embedding(\n    len(tokenizer.word_index) + 1,\n    embed_size_short,\n    embeddings_initializer = tf.keras.initializers.Constant(embedding_matrix),\n    trainable = False\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:03:41.306032Z","iopub.execute_input":"2022-02-16T14:03:41.306493Z","iopub.status.idle":"2022-02-16T14:04:11.412027Z","shell.execute_reply.started":"2022-02-16T14:03:41.306453Z","shell.execute_reply":"2022-02-16T14:04:11.411241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# encoding question input\nquestion_input = Input(shape=(None,))\nquestion_x = embedding_layer2(question_input)\nquestion_x = SpatialDropout1D(0.2)(question_x)\nquestion_x = Bidirectional(LSTM(200, return_sequences=True))(question_x)\nquestion_x = Bidirectional(LSTM(100, return_sequences=True))(question_x)\n\n# encoding answer input\nanswer_input = Input(shape=(None,))\nanswer_x = embedding_layer2(answer_input)\nanswer_x = SpatialDropout1D(0.2)(answer_x)\nanswer_x = Bidirectional(LSTM(250, return_sequences=True))(answer_x)\nanswer_x = Bidirectional(LSTM(150, return_sequences=True))(answer_x)\n\n# merge the encodings\ncombined_x = concatenate([question_x, answer_x])\n\n# predict start index of the short answer\nstart_x = Dropout(0.1)(combined_x) \nstart_x = Conv1D(1,1)(start_x)\nstart_x = Flatten()(start_x)\nstart_x = Activation('softmax', name='start_token')(start_x)\n\n# predict end index of the short answer\nend_x = Dropout(0.1)(combined_x) \nend_x = Conv1D(1,1)(end_x)\nend_x = Flatten()(end_x)\nend_x = Activation('softmax', name='end_token')(end_x)\n\n# merge the parts into one model\nshort_model = tf.keras.models.Model(inputs=[answer_input, question_input], outputs=[start_x, end_x])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:04:19.464290Z","iopub.execute_input":"2022-02-16T14:04:19.465088Z","iopub.status.idle":"2022-02-16T14:04:21.542375Z","shell.execute_reply.started":"2022-02-16T14:04:19.465043Z","shell.execute_reply":"2022-02-16T14:04:21.541673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_model.compile(\n    loss='categorical_crossentropy', \n    optimizer='adam',\n    metrics=['categorical_accuracy', 'Recall', 'Precision'])","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:04:24.511909Z","iopub.execute_input":"2022-02-16T14:04:24.512456Z","iopub.status.idle":"2022-02-16T14:04:24.524309Z","shell.execute_reply.started":"2022-02-16T14:04:24.512417Z","shell.execute_reply":"2022-02-16T14:04:24.523592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_model.summary()","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:04:27.165468Z","iopub.execute_input":"2022-02-16T14:04:27.166298Z","iopub.status.idle":"2022-02-16T14:04:27.181813Z","shell.execute_reply.started":"2022-02-16T14:04:27.166250Z","shell.execute_reply":"2022-02-16T14:04:27.180924Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# define callbacks for the short model\n#increased patience or number of epochs with no improvement after which training will be stopped.\ncallbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=4, verbose=1),\n    tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1),]","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:04:31.063767Z","iopub.execute_input":"2022-02-16T14:04:31.064069Z","iopub.status.idle":"2022-02-16T14:04:31.069874Z","shell.execute_reply.started":"2022-02-16T14:04:31.064032Z","shell.execute_reply":"2022-02-16T14:04:31.068130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = short_model.fit(\n    x = [train_long_answers, train_questions], \n    y = [train_start_labels, train_end_labels],epochs = short_epochs, callbacks = callbacks,\n    validation_data = ([test_long_answers, test_questions], [test_start_labels, test_end_labels]),\n    batch_size = short_batch_size,  shuffle = True)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:04:33.424579Z","iopub.execute_input":"2022-02-16T14:04:33.424831Z","iopub.status.idle":"2022-02-16T14:08:21.516435Z","shell.execute_reply.started":"2022-02-16T14:04:33.424802Z","shell.execute_reply":"2022-02-16T14:08:21.515785Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_model.save('short_model.h5')","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:11:45.485287Z","iopub.execute_input":"2022-02-16T14:11:45.485923Z","iopub.status.idle":"2022-02-16T14:12:17.826537Z","shell.execute_reply.started":"2022-02-16T14:11:45.485884Z","shell.execute_reply":"2022-02-16T14:12:17.825711Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Model Evaluation**","metadata":{}},{"cell_type":"markdown","source":"identifying if correct short answers given to a long answers","metadata":{}},{"cell_type":"code","source":"print('Epoch: {0}'.format(len(history.history['loss'])))\nprint('Loss: {0}'.format(history.history['loss'][-1]))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:17.828262Z","iopub.execute_input":"2022-02-16T14:12:17.828525Z","iopub.status.idle":"2022-02-16T14:12:17.840960Z","shell.execute_reply.started":"2022-02-16T14:12:17.828489Z","shell.execute_reply":"2022-02-16T14:12:17.839348Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Training final results')\n\naccuracy = history.history['start_token_categorical_accuracy'][-1]\nrecall = history.history['start_token_recall'][-1]\nprecision = history.history['start_token_precision'][-1]\n\nprint('--------------------------------------------------')\nprint('Start token accuracy: {0}'.format(accuracy))\nprint('Start token recall: {0}'.format(recall))\nprint('Start token precision: {0}'.format(precision))\nprint('Start token F1 score: {0:.4f}'.format(2 * (precision * recall) / (precision + recall)))\n\nprint('--------------------------------------------------')\n\naccuracy = history.history['end_token_categorical_accuracy'][-1]\nrecall = history.history['end_token_recall_1'][-1]\nprecision = history.history['end_token_precision_1'][-1]\n\nprint('End token accuracy: {0}'.format(accuracy))\nprint('End token recall: {0}'.format(recall))\nprint('End token precision: {0}'.format(precision))\nprint('End token F1 score: {0:.4f}'.format(2 * (precision * recall) / (precision + recall)))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:21.804807Z","iopub.execute_input":"2022-02-16T14:12:21.805103Z","iopub.status.idle":"2022-02-16T14:12:21.818127Z","shell.execute_reply.started":"2022-02-16T14:12:21.805072Z","shell.execute_reply":"2022-02-16T14:12:21.817345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Validation final results')\nprint('--------------------------------------------------')\n\naccuracy = history.history['val_start_token_categorical_accuracy'][-1]\nrecall = history.history['val_start_token_recall'][-1]\nprecision = history.history['val_start_token_precision'][-1]\n\nprint('Start token accuracy: {0}'.format(accuracy))\nprint('Start token recall: {0}'.format(recall))\nprint('Start token precision: {0}'.format(precision))\nprint('Start token F1 score: {0:.4f}'.format( 2 * (precision * recall) / (precision + recall)))\n\nprint('--------------------------------------------------')\n\naccuracy = history.history['val_end_token_categorical_accuracy'][-1]\nrecall = history.history['val_end_token_recall_1'][-1]\nprecision = history.history['val_end_token_precision_1'][-1]\n\nprint('End token accuracy: {0}'.format(accuracy))\nprint('End token recall: {0}'.format(recall))\nprint('End token precision: {0}'.format(precision))\nprint('End token F1 score: {0:.4f}'.format(2 * (precision * recall) / (precision + recall)))","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:25.364871Z","iopub.execute_input":"2022-02-16T14:12:25.365456Z","iopub.status.idle":"2022-02-16T14:12:25.375861Z","shell.execute_reply.started":"2022-02-16T14:12:25.365418Z","shell.execute_reply":"2022-02-16T14:12:25.375103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Evaluate plotting training & validation loss\n\nfig, ax = plt.subplots(1, 2, figsize=(15, 5))\n\nax[0].set_title('Training Loss')\nax[0].plot(history.history['loss'],marker = '.',mec = 'r', mfc = 'r')\n\nax[1].set_title('Validation Loss')\nax[1].plot(history.history['val_loss'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")\n","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:28.225110Z","iopub.execute_input":"2022-02-16T14:12:28.225671Z","iopub.status.idle":"2022-02-16T14:12:28.532668Z","shell.execute_reply.started":"2022-02-16T14:12:28.225633Z","shell.execute_reply":"2022-02-16T14:12:28.531858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plotting metrics; accuracy, precision, recall for the first output array\nfig, ax = plt.subplots(3, 2, figsize=(15, 10))\n\nfig.suptitle('First output-start token')\n\nax[0,0].set_title('Training Accuracy')\nax[0,0].plot(history.history['start_token_categorical_accuracy'],marker = '.',mec = 'r', mfc = 'r')\n\nax[0,1].set_title('Validation Accuracy')\nax[0,1].plot(history.history['val_start_token_categorical_accuracy'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")\n\nax[1,0].set_title('Training Recall')\nax[1,0].plot(history.history['start_token_recall'],marker = '.',mec = 'r', mfc = 'r')\n\nax[1,1].set_title('Validation Recall')\nax[1,1].plot(history.history['val_start_token_recall'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")\n\nax[2,0].set_title('Training Precision')\nax[2,0].plot(history.history['start_token_precision'],marker = '.',mec = 'r', mfc = 'r')\n\nax[2,1].set_title('Validation Precision')\nax[2,1].plot(history.history['val_start_token_precision'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:31.085731Z","iopub.execute_input":"2022-02-16T14:12:31.086025Z","iopub.status.idle":"2022-02-16T14:12:31.811490Z","shell.execute_reply.started":"2022-02-16T14:12:31.085977Z","shell.execute_reply":"2022-02-16T14:12:31.810841Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#plotting metrics; accuracy, precision, recall for the second output array\nfig, ax = plt.subplots(3, 2, figsize=(15, 10))\n\nfig.suptitle('Second output-end token')\n\nax[0,0].set_title('Training Accuracy')\nax[0,0].plot(history.history['end_token_categorical_accuracy'],marker = '.',mec = 'r', mfc = 'r')\n\nax[0,1].set_title('Validation Accuracy')\nax[0,1].plot(history.history['val_end_token_categorical_accuracy'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")\n\nax[1,0].set_title('Training Recall')\nax[1,0].plot(history.history['end_token_recall_1'],marker = '.',mec = 'r', mfc = 'r')\n\nax[1,1].set_title('Validation Recall')\nax[1,1].plot(history.history['val_end_token_recall_1'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")\n\nax[2,0].set_title('Training Precision')\nax[2,0].plot(history.history['end_token_precision_1'],marker = '.',mec = 'r', mfc = 'r')\n\nax[2,1].set_title('Validation Precision')\nax[2,1].plot(history.history['val_end_token_precision_1'],marker = '.',mec = 'r', mfc = 'r',color=\"green\")","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:35.174103Z","iopub.execute_input":"2022-02-16T14:12:35.174680Z","iopub.status.idle":"2022-02-16T14:12:35.960523Z","shell.execute_reply.started":"2022-02-16T14:12:35.174642Z","shell.execute_reply":"2022-02-16T14:12:35.959844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#testing with live inputs\ndef test_short_answer(question, long_answer):\n    sentences = [long_answer, question]\n    \n    sentences = encode(sentences, tokenizer)\n    \n    predictions = short_model.predict(\n        [np.expand_dims(sentences[0], axis=0), np.expand_dims(sentences[1], axis=0)])\n    \n    predictions = np.array(predictions)\n    \n    prediction_start = np.argmax(predictions[0,0])\n    prediction_end = np.argmax(predictions[1,0])\n    prediction_string = ' '.join(long_answer.split(' ')[prediction_start:prediction_end])\n\n    return prediction_start, prediction_end, prediction_string","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:39.616151Z","iopub.execute_input":"2022-02-16T14:12:39.617079Z","iopub.status.idle":"2022-02-16T14:12:39.625218Z","shell.execute_reply.started":"2022-02-16T14:12:39.617025Z","shell.execute_reply":"2022-02-16T14:12:39.624280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"question = 'which is the most common use of opt-in e-mail marketing'\nlong_answer = \"<P> A common example of permission marketing is a newsletter sent to an advertising firm 's customers . Such newsletters inform customers of upcoming events or promotions , or new products . In this type of advertising , a company that wants to send a newsletter to their customers may ask them at the point of purchase if they would like to receive the newsletter . </P>\"","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:42.054782Z","iopub.execute_input":"2022-02-16T14:12:42.055083Z","iopub.status.idle":"2022-02-16T14:12:42.059420Z","shell.execute_reply.started":"2022-02-16T14:12:42.055048Z","shell.execute_reply":"2022-02-16T14:12:42.058570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start, end, short_answer = test_short_answer(question, long_answer)\n\nprint('Start token: ' + str(start))\nprint('End token: ' + str(end))\nprint('Answer: ' + short_answer)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:44.943644Z","iopub.execute_input":"2022-02-16T14:12:44.944231Z","iopub.status.idle":"2022-02-16T14:12:46.971153Z","shell.execute_reply.started":"2022-02-16T14:12:44.944186Z","shell.execute_reply":"2022-02-16T14:12:46.970338Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"question = 'who invented a portable handset'\nlong_answer=\"<P> Martin Cooper invented a portable handset in 1973, when he was a project manager at Motorola. It was almost three decades after the idea of cellular communications was introduced by Bell Laboratories. </P>\"","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:49.645616Z","iopub.execute_input":"2022-02-16T14:12:49.646329Z","iopub.status.idle":"2022-02-16T14:12:49.652116Z","shell.execute_reply.started":"2022-02-16T14:12:49.646276Z","shell.execute_reply":"2022-02-16T14:12:49.650712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start, end, short_answer = test_short_answer(question, long_answer)\n\nprint('Start token: ' + str(start))\nprint('End token: ' + str(end))\nprint('Answer: ' + short_answer)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:53.604232Z","iopub.execute_input":"2022-02-16T14:12:53.604527Z","iopub.status.idle":"2022-02-16T14:12:53.710592Z","shell.execute_reply.started":"2022-02-16T14:12:53.604494Z","shell.execute_reply":"2022-02-16T14:12:53.709761Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"question = 'who is the south african high commissioner in london'\nlong_answer = \"<P> Tracy McConnell , better known as `` The Mother '' , is the title character from the CBS television sitcom How I Met Your Mother . The show , narrated by Future Ted , tells the story of how Ted Mosby met The Mother . Tracy McConnell appears in 8 episodes from `` Lucky Penny '' to `` The Time Travelers '' as an unseen character ; she was first seen fully in `` Something New '' and was promoted to a main character in season 9 . The Mother is played by Cristin Milioti . </P>\"","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:12:58.043747Z","iopub.execute_input":"2022-02-16T14:12:58.044291Z","iopub.status.idle":"2022-02-16T14:12:58.048105Z","shell.execute_reply.started":"2022-02-16T14:12:58.044251Z","shell.execute_reply":"2022-02-16T14:12:58.047181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"start, end, short_answer = test_short_answer(question, long_answer)\n\nprint('Start token: ' + str(start))\nprint('End token: ' + str(end))\nprint('Answer: ' + short_answer)","metadata":{"execution":{"iopub.status.busy":"2022-02-16T14:13:01.303901Z","iopub.execute_input":"2022-02-16T14:13:01.304479Z","iopub.status.idle":"2022-02-16T14:13:01.895663Z","shell.execute_reply.started":"2022-02-16T14:13:01.304442Z","shell.execute_reply":"2022-02-16T14:13:01.894753Z"},"trusted":true},"execution_count":null,"outputs":[]}]}