{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport json\nimport gc\nimport sys\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"bs_path = '/kaggle/input/tensorflow2-question-answering/'\ntrain_file = 'simplified-nq-train.jsonl'\ntest_file = 'simplified-nq-test.jsonl'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_data(file_name, num_records = sys.maxsize): # = sys.maxsize\n    current_record = 1\n    records = []\n    \n    with open(os.path.join(bs_path, file_name)) as file:\n        line = file.readline()\n        while(line):\n            records.append(json.loads(line))\n            line = file.readline()\n            if current_record > num_records:\n                break\n                \n            if current_record % 5000 == 0:\n                print(current_record)\n                gc.collect()\n                \n            current_record = current_record + 1\n    df = pd.DataFrame(records)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmax_records = 5000\ndf_train = read_data(train_file, max_records)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['question_text'][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_train['document_text'][0:3]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_train['long_answer_candidates'][0][54]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_train['document_text'][0]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_train['annotations'][0]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['yes_no_answer'] = [item[0]['yes_no_answer'] for item in df_train['annotations']]\ndf_train['long_answer'] = [item[0]['long_answer'] for item in df_train['annotations']]\ndf_train['short_answers'] = [item[0]['short_answers'] for item in df_train['annotations']]\ndf_train['annotation_id'] = [item[0]['annotation_id'] for item in df_train['annotations']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['yes_no_answer'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Getting values out of annotations","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Short answer\nstart_vals = []\nend_vals = []\n\nfor item in df_train['short_answers']:\n    start = -1\n    end = -1\n    if len(item) > 0:\n        start = item[0]['start_token']\n        end = item[0]['end_token']\n    #if len(item) > 1: #TODO -> there are cases with more than one correct long/short answers, handle/check it\n    #    print(item)\n    start_vals.append(start)\n    end_vals.append(end)\ndf_train['short_answer_start'] = start_vals\ndf_train['short_answer_end'] = end_vals\n\n# del df_train['short_answers'] #TODO","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Long answer\n    \ndf_train['long_answer_start'] = [item['start_token'] for item in df_train['long_answer']]\ndf_train['long_answer_end'] = [item['end_token'] for item in df_train['long_answer']]\ndf_train['long_answer_index'] = [item['candidate_index'] for item in df_train['long_answer']]\n\n# del df_train['long_answer'] #TODO","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_train.head()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_train.isnull().sum()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_train.head()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train[['document_text', 'question_text', 'short_answer_start', 'short_answer_end']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_train.head(1)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['document_text'][0].split()[1955:1969]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndf_train['document_text'][0].split()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## MODEL ARCHITECTURE\n\n#inputs = [['wikipedia', 'dcoument', 'text', '...'], ['question', 'text', '...']]\ninputs = [['wikipedia', 'dcoument', 'text', '...']]\noutputs = [0,       1, 1, '...']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndoc_text = df_train['document_text'][0]\nshort_answer_start = df_train['short_answer_start'][0]\nshort_answer_end = df_train['short_answer_end'][0]\n\nque_text = df_train['question_text'][0]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\n\nmax_len_document = 2500\nmax_len_que = 50\nmax_len_word = 40\nmax_len_input = max_len_document + max_len_que\n\ndoc_text_lst = []\ndoc_text_lst.append(doc_text.split())\n\nque_text_lst = []\nque_text_lst.append(que_text.split())\n\noutput_label = np.zeros(max_len_document)\n\ndoc_text_lst = pad_sequences(doc_text_lst, maxlen=max_len_document, dtype=object, padding='post', truncating='post', value='')\n\nque_text_lst = pad_sequences(que_text_lst, maxlen=max_len_que, dtype=object, padding='post', truncating='post', value='')\n\nif short_answer_end <= max_len_document:\n    output_label[short_answer_start:short_answer_end] = np.ones(short_answer_end - short_answer_start)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing import text\ndef train_tokenizer(train_data):\n    tokenizer = text.Tokenizer(num_words=50, filters='!\"#$%&()*+,-.:;=?@[\\\\]^_`{|}~\\t\\n', lower=True, char_level=True) #split='', \n    tokenizer.fit_on_texts(train_data)\n    return tokenizer\n\ntokenizer = train_tokenizer(doc_text_lst[0])\ndoc_text_lst[0] = tokenizer.texts_to_sequences(doc_text_lst[0])\ndoc_text_chars = pad_sequences(doc_text_lst[0], maxlen=max_len_word, padding='post', truncating='post', value=0)\n\nque_text_lst[0] = tokenizer.texts_to_sequences(que_text_lst[0])\nque_text_chars = pad_sequences(que_text_lst[0], maxlen=max_len_word, padding='post', truncating='post', value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"doc_text_chars","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndoc_text_chars.shape\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"que_text_chars","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nque_text_chars.shape\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = np.array(doc_text_chars)\n\nlst = []\n\nlst.append([[item] for item in doc_text_chars])\nlst.append([[item] for item in doc_text_chars])\nx_train = np.asarray(lst)\n\ny_train = []\ny_train.append([[item] for item in output_label])\ny_train.append([[item] for item in output_label])\ny_train = np.asarray(y_train)\n\n# history = model.fit(x_train, y_train, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nx_train.shape\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"doc_input = keras.Input(shape=(max_len_document, 1, 40), name='doc_text')  #TODO -> make the length of the sequences variable\n\nbody_features = doc_input\n\nbody_features = layers.Reshape((max_len_document, 40))(body_features)\n\n#Embed each character in the text into a 64-dimensional vector\nbody_features = layers.Embedding(50, 64)(body_features)\n\nbody_features = layers.TimeDistributed(layers.LSTM(64))(body_features)\n\nshort_answer = layers.TimeDistributed(layers.Dense(2, activation='softmax', name='short_answer'))(body_features)\n# Instantiate an end-to-end model predicting both priority and department\nmodel = keras.Model(inputs=doc_input, outputs=short_answer, name='qa_model')\nmodel.compile(loss='sparse_categorical_crossentropy', optimizer=keras.optimizers.RMSprop(), metrics=['accuracy'])\nmodel.summary()\n\nhistory = model.fit(x_train, y_train, epochs=3)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"doc_input = keras.Input(shape=(2500, 1, 40), name='doc_text')  #TODO -> make the length of the sequences variable\n\nbody_features = doc_input\n\nbody_features = layers.TimeDistributed(layers.LSTM(64))(body_features)\n\nshort_answer = layers.TimeDistributed(layers.Dense(2, activation='softmax', name='short_answer'))(body_features)\n# Instantiate an end-to-end model predicting both priority and department\nmodel = keras.Model(inputs=doc_input, outputs=short_answer, name='qa_model')\nmodel.compile(loss='sparse_categorical_crossentropy', optimizer=keras.optimizers.RMSprop(), metrics=['accuracy'])\nmodel.summary()\n\nhistory = model.fit(x_train, y_train, epochs=10)\n\n# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#V2 ready to go\nx_train = np.array(doc_text_chars)\n\nlst = []\nlst.append(doc_text_chars)\nlst.append(doc_text_chars)\nx_train = np.asarray(lst)\n\n\ny_train = []\ny_train.append(1950)\ny_train.append(1950)\ny_train = np.asarray(y_train)\n\n# history = model.fit(x_train, y_train, epochs=5)\n\ndoc_input = keras.Input(shape=(2500, 40), name='doc_text')  #TODO -> make the length of the sequences variable\n\nbody_features = doc_input\n\n#Embed each character in the text into a 64-dimensional vector\nbody_features = layers.Embedding(50, 64)(body_features)\n\n# concolution 2d in order to process the input further\nbody_features = layers.Conv2D(32, 3, activation='relu')(body_features)\n\nbody_features = layers.Reshape((2498, 38 * 32))(body_features)\n\n# Reduce sequence of embedded words in the body into a single 32-dimensional vector\nbody_features = layers.LSTM(64)(body_features)\n# Stick a department classifier on top of the features\ndepartment_pred = layers.Dense(2500, activation='softmax', name='department')(body_features)\n# Instantiate an end-to-end model predicting both priority and department\nmodel = keras.Model(inputs=doc_input, outputs=department_pred, name='qa_model')\nmodel.compile(loss='sparse_categorical_crossentropy', optimizer=keras.optimizers.RMSprop(), metrics=['accuracy'])\nmodel.summary()\n\nhistory = model.fit(x_train, y_train, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}