{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import packages"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport json\nimport gc\nimport sys\n\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"base_path = '/kaggle/input/tensorflow2-question-answering/'\ntrain_file_name = 'simplified-nq-train.jsonl'\ntest_file_name = 'simplified-nq-test.jsonl'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read Data In"},{"metadata":{"trusted":true},"cell_type":"code","source":"def read_data(file_name, num_records = sys.maxsize): # = sys.maxsize\n    current_record = 1\n    records = []\n    \n    with open(os.path.join(base_path, file_name)) as file:\n        line = file.readline()\n        while(line):\n            records.append(json.loads(line))\n            line = file.readline()\n            if current_record > num_records:\n                break\n                \n            if current_record % 5000 == 0:\n                print(current_record)\n                gc.collect()\n                \n            current_record = current_record + 1\n    df = pd.DataFrame(records)\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nmax_records = 10000\ndf_train = read_data(train_file_name, max_records)\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['question_text'][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['document_text'][0:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['long_answer_candidates'][0][54]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['document_text'][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['annotations'][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['yes_no_answer'] = [item[0]['yes_no_answer'] for item in df_train['annotations']]\ndf_train['long_answer'] = [item[0]['long_answer'] for item in df_train['annotations']]\ndf_train['short_answers'] = [item[0]['short_answers'] for item in df_train['annotations']]\ndf_train['annotation_id'] = [item[0]['annotation_id'] for item in df_train['annotations']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['yes_no_answer'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Getting values out of annotations"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Short answer\nstart_vals = []\nend_vals = []\n\nfor item in df_train['short_answers']:\n    start = -1\n    end = -1\n    if len(item) > 0:\n        start = item[0]['start_token']\n        end = item[0]['end_token']\n    #if len(item) > 1: #TODO -> there are cases with more than one correct long/short answers, handle/check it\n    #    print(item)\n    start_vals.append(start)\n    end_vals.append(end)\ndf_train['short_answer_start'] = start_vals\ndf_train['short_answer_end'] = end_vals\n\n# del df_train['short_answers'] #TODO","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Long answer\n    \ndf_train['long_answer_start'] = [item['start_token'] for item in df_train['long_answer']]\ndf_train['long_answer_end'] = [item['end_token'] for item in df_train['long_answer']]\ndf_train['long_answer_index'] = [item['candidate_index'] for item in df_train['long_answer']]\n\n# del df_train['long_answer'] #TODO","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df_train[['document_text', 'question_text', 'short_answer_start', 'short_answer_end', 'annotation_id']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['document_text'][0].split()[1955:1969]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## HOW TO MODEL THE PROBLEM/architectur\n#TRAIN =>\n#[..., 'example', 'of', 'permission', 'marketing', 'is', 'a', 'newsletter', 'sent', 'to', 'an', 'advertising', 'firm', \"'s\", 'customers', ...]\n#[..., 0,          0,        0,            0,        0,   1,         1,        1,     1,    1,        1,          1,     1,      1      , ...]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# TEST -> possible problem (s) !\n\n# What to do with this scenario:\n# [..., 'example', 'of', 'permission', 'marketing', 'is', 'a', 'newsletter', 'sent', 'to', 'an', 'advertising', 'firm', \"'s\", 'customers', ...]\n# [..., 1,          1,        1,            0,        0,   1,         1,        1,     1,    0,        0,          1,     1,      1      , ...]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['document_text'][0].split()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## MODEL architecture"},{"metadata":{"trusted":true},"cell_type":"code","source":"#inputs = [['wikipedia', 'dcoument', 'text', '...'], ['question', 'text', '...']]\ninputs = [['wikipedia', 'dcoument', 'text', '...']]\noutputs = [0,       1, 1, '...']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ### This code works for individual samples, now we will try to focus on having the same for the entire dataset\n\n# doc_text = df_train['document_text'][0]\n# short_answer_start = df_train['short_answer_start'][0]\n# short_answer_end = df_train['short_answer_end'][0]\n\n# que_text = df_train['question_text'][0]\n\n# from keras.preprocessing.sequence import pad_sequences\n\n# max_len_document = 2500\n# max_len_que = 50\n# max_len_word = 40\n# max_len_input = max_len_document + max_len_que\n\n# doc_text_lst = []\n# doc_text_lst.append(doc_text.split())\n\n# que_text_lst = []\n# que_text_lst.append(que_text.split())\n\n# output_label = np.zeros(max_len_input)\n\n# doc_text_lst = pad_sequences(doc_text_lst, maxlen=max_len_document, dtype=object, padding='post', truncating='post', value='')\n\n# que_text_lst = pad_sequences(que_text_lst, maxlen=max_len_que, dtype=object, padding='post', truncating='post', value='')\n\n# if short_answer_end <= max_len_document:\n#     output_label[short_answer_start:short_answer_end] = np.ones(short_answer_end - short_answer_start)\n    \n    \n# from keras.preprocessing import text\n# def train_tokenizer(train_data):\n#     tokenizer = text.Tokenizer(num_words=50, filters='!\"#$%&()*+,-.:;=?@[\\\\]^_`{|}~\\t\\n', lower=True, char_level=True) #split='', \n#     tokenizer.fit_on_texts(train_data)\n#     return tokenizer\n\n# tokenizer = train_tokenizer(doc_text_lst[0])\n# doc_text_lst[0] = tokenizer.texts_to_sequences(doc_text_lst[0])\n# doc_text_chars = pad_sequences(doc_text_lst[0], maxlen=max_len_word, padding='post', truncating='post', value=0)\n\n# que_text_lst[0] = tokenizer.texts_to_sequences(que_text_lst[0])\n# que_text_chars = pad_sequences(que_text_lst[0], maxlen=max_len_word, padding='post', truncating='post', value=0)\n\n\n# x_train = np.array(doc_text_chars)\n\n# lst = []\n\n\n# val_temp = [[item] for item in doc_text_chars]\n# _ = [val_temp.append([item]) for item in que_text_chars]\n\n# lst.append(val_temp)\n# lst.append(val_temp)\n# x_train = np.asarray(lst)\n\n# y_train = []\n# y_train.append([[item] for item in output_label])\n# y_train.append([[item] for item in output_label])\n# y_train = np.asarray(y_train)\n\n# # history = model.fit(x_train, y_train, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for all the samples -> continue from here\n\ndoc_text = df_train['document_text']\nshort_answer_start = df_train['short_answer_start']\nshort_answer_end = df_train['short_answer_end']\nque_text = df_train['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nmax_len_document = 1000\nmax_len_que = 50\nmax_len_word = 10\nmax_len_input = max_len_document + max_len_que\nmax_len_input = 1000 ##### TEMP","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# for item in \noutput_labels = []\nfor index in short_answer_end.index:\n    output_label = np.zeros(max_len_input)\n     \n    end = short_answer_end.iloc[index]\n    start = short_answer_start.iloc[index]\n        \n    if end > -1 and end <= max_len_document:\n        output_label[start:end] = np.ones(end - start)\n    output_labels.append(output_label)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing import text\ndef train_tokenizer(train_data):\n    tokenizer = text.Tokenizer(num_words=50, filters='!\"#$%&()*+,-.:;=?@[\\\\]^_`{|}~\\t\\n', lower=True, char_level=True) #split='', \n    tokenizer.fit_on_texts(train_data)\n    return tokenizer\n\ntokenizer = train_tokenizer(doc_text[0:min(800, doc_text.shape[0])])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"doc_text.shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from keras.preprocessing.sequence import pad_sequences\n\n# max_len_document = 2500\n# max_len_que = 50\n# max_len_word = 40\n# max_len_input = max_len_document + max_len_que\n\n# doc_text_lst = []\n\n# for item in doc_text:\n#     tmp = item.split()\n#     doc_text_lst.append(tokenizer.texts_to_sequences(tmp))\n\n# que_text_lst = []\n\n# for item in que_text:\n#     tmp = item.split()\n#     que_text_lst.append(tokenizer.texts_to_sequences(tmp))\n    \n# doc_text_lst = pad_sequences(doc_text_lst, maxlen=max_len_document, dtype=object, padding='post', truncating='post', value='')\n# que_text_lst = pad_sequences(que_text_lst, maxlen=max_len_que, dtype=object, padding='post', truncating='post', value='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfrom keras.preprocessing.sequence import pad_sequences\n\ndoc_text_lst = []\n\nfor item in doc_text:\n    doc_text_lst.append(item.split())\n\nque_text_lst = []\n\nfor item in que_text:\n    que_text_lst.append(item.split())\n    \ndoc_text_lst = pad_sequences(doc_text_lst, maxlen=max_len_document, dtype=object, padding='post', truncating='post', value='')\nque_text_lst = pad_sequences(que_text_lst, maxlen=max_len_que, dtype=object, padding='post', truncating='post', value='')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndoc_text_chars = []\nfor i in range(doc_text_lst.shape[0]):\n    tmp = tokenizer.texts_to_sequences(doc_text_lst[i])\n    doc_text_chars.append(pad_sequences(tmp, maxlen=max_len_word, padding='post', truncating='post', value=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nque_text_chars = []\nfor i in range(que_text_lst.shape[0]):\n    tmp = tokenizer.texts_to_sequences(que_text_lst[i])\n    que_text_chars.append(pad_sequences(tmp, maxlen=max_len_word, padding='post', truncating='post', value=0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"doc_text_lst = None\nque_text_lst = None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = np.array(doc_text_chars)\n# TODO -> look into this part later\n\n# lst = []\n\n# val_temp = [[item] for item in doc_text_chars]\n# _ = [val_temp.append([item]) for item in que_text_chars]\n\n# lst.append(val_temp)\n# lst.append(val_temp)\n# x_train = np.asarray(lst)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = []\ny_train = [[[item2] for item2 in item] for item in output_labels]\ny_train = np.asarray(y_train)\n\n# history = model.fit(x_train, y_train, epochs=5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's join question with document text to make it a single inout to the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nfrom keras import losses\ndoc_input = keras.Input(shape=(max_len_input, max_len_word), name='doc_text')  #TODO -> make the length of the sequences variable\n\nbody_features = doc_input\n\n# body_features = layers.Reshape((max_len_input, 40))(body_features)\n\n#Embed each character in the text into a 64-dimensional vector\nbody_features = layers.Embedding(50, 10)(body_features)\n\nbody_features = layers.TimeDistributed(layers.LSTM(25))(body_features)\n\nshort_answer = layers.TimeDistributed(layers.Dense(1, activation='sigmoid', name='short_answer'))(body_features)\n# Instantiate an end-to-end model predicting both priority and department\nmodel = keras.Model(inputs=doc_input, outputs=short_answer, name='qa_model')\nmodel.compile(loss= losses.binary_crossentropy\n, optimizer=keras.optimizers.RMSprop(), metrics=['accuracy'])\nmodel.summary()\n\nhistory = model.fit(x_train, y_train, epochs=5, validation_split = 0.2)\n\n# model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_scores = model.evaluate(x_test, y_test, verbose=2)\n# print('Test loss:', test_scores[0])\n# print('Test accuracy:', test_scores[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Covert the outout into submission file"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = x_train # for now\ntest_scores = model.predict(x_test, verbose=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_scores","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in test_scores[0]:\n    if i > 0.5:\n        print(i[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_short_answer(single_output):\n    answer_start = -1\n    answer_end = -1\n    i = 0\n    for item in single_output:\n        if item[0] > 0.5:\n            if answer_start == -1:\n                answer_start = i \n                answer_end = i\n            else:\n                answer_end = i\n        elif answer_start != -1 :\n            break\n        i = i + 1\n    return answer_start, answer_end","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_scores.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for item in test_scores:\n    answer_start, answer_end = get_short_answer(item)\n    print(answer_start, answer_end)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"short_answers = []\nlong_answers = []\nexample_id = []\nfor annotation_id in df_test['annotation_id']:\n    example_id.append('-' + str(annotation_id) + '_short')\n    example_id.append('-' + str(annotation_id) + '_long')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"-7853356005143141653_short,YES","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}