{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Packages**","metadata":{}},{"cell_type":"code","source":"import json\nfrom tqdm.notebook import tqdm\n\nimport numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import LSTM, Bidirectional, GlobalMaxPooling1D, SpatialDropout1D, Dense, Dropout, Input, concatenate, Conv1D, Activation, Flatten\n\nfrom nltk.corpus import stopwords\nimport re\n","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:21:54.7692Z","iopub.execute_input":"2021-10-07T21:21:54.769543Z","iopub.status.idle":"2021-10-07T21:22:00.343688Z","shell.execute_reply.started":"2021-10-07T21:21:54.769512Z","shell.execute_reply":"2021-10-07T21:22:00.342794Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Constants Definiation**","metadata":{}},{"cell_type":"code","source":"# data to load\nNUM_OF_TRAIN_QUESTIONS = 1000\nNUM_OF_VAL_QUESTIONS = 1050\nSAMPLE_RATE = 15\nTRAIN_PATH = '../input/tensorflow2-question-answering/simplified-nq-train.jsonl'\n\n\n# long answer model parameters\nEPOCHS = 80\nBATCH_SIZE = 16\nEMBED_SIZE = 100\nCLASS_WEIGHTS = {0: 0.5, 1: 5.}\n\n# short answer model parameters\nSHORT_EPOCHS = 80\nSHORT_BATCH_SIZE = 16\nSHORT_EMBED_SIZE = 200","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:00.345144Z","iopub.execute_input":"2021-10-07T21:22:00.345513Z","iopub.status.idle":"2021-10-07T21:22:00.34997Z","shell.execute_reply.started":"2021-10-07T21:22:00.345465Z","shell.execute_reply":"2021-10-07T21:22:00.349192Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **Reading The Data Set From Json files**","metadata":{}},{"cell_type":"markdown","source":"**reading data helper funcs**","metadata":{}},{"cell_type":"code","source":"def get_line_of_data(file):\n    line = file.readline()\n    line = json.loads(line)\n    \n    return line\n\n\ndef get_question_and_document(line):\n    question = line['question_text']\n    text = line['document_text'].split(' ')\n    annotations = line['annotations'][0]\n    return question, text, annotations\n                \n                \ndef get_long_candidate(i, annotations, candidate):\n    # check if this candidate is the correct answer\n    if i == annotations['long_answer']['candidate_index']:\n        label = True\n    else:\n        label = False\n\n    # get place where long answer starts and ends in the document text\n    long_start = candidate['start_token']\n    long_end = candidate['end_token']\n    \n    return label, long_start, long_end\n\n#create dataset with two features (question and long_answer)\ndef form_data_row(question, label, text, long_start, long_end):\n    row = {\n        'question': question,\n        'long_answer': ' '.join(text[long_start:long_end]),\n        'is_long_answer': label,\n    }\n    \n    return row\n\n\ndef load_data(file_path, questions_start, questions_end):\n    rows = []\n    \n    with open(file_path) as file:\n\n        for i in tqdm(range(questions_start, questions_end)):\n            line = get_line_of_data(file)\n            \n            question, text, annotations = get_question_and_document(line)\n\n            for i, candidate in enumerate(line['long_answer_candidates']):\n                label, long_start, long_end = get_long_candidate(i, annotations, candidate)\n\n                if label == True or (i % SAMPLE_RATE == 0):##?? samplerate\n                    rows.append(\n                        form_data_row(question, label, text, long_start, long_end)\n                    )\n        \n    return pd.DataFrame(rows)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:00.351763Z","iopub.execute_input":"2021-10-07T21:22:00.352237Z","iopub.status.idle":"2021-10-07T21:22:00.365256Z","shell.execute_reply.started":"2021-10-07T21:22:00.352202Z","shell.execute_reply":"2021-10-07T21:22:00.364367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_df = load_data(TRAIN_PATH, NUM_OF_TRAIN_QUESTIONS, NUM_OF_VAL_QUESTIONS)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:00.366856Z","iopub.execute_input":"2021-10-07T21:22:00.367192Z","iopub.status.idle":"2021-10-07T21:22:00.478146Z","shell.execute_reply.started":"2021-10-07T21:22:00.367158Z","shell.execute_reply":"2021-10-07T21:22:00.477345Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:00.479356Z","iopub.execute_input":"2021-10-07T21:22:00.479852Z","iopub.status.idle":"2021-10-07T21:22:00.501054Z","shell.execute_reply.started":"2021-10-07T21:22:00.479813Z","shell.execute_reply":"2021-10-07T21:22:00.500309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = load_data(TRAIN_PATH, 0, NUM_OF_TRAIN_QUESTIONS)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:40:05.852575Z","iopub.execute_input":"2021-09-28T19:40:05.852884Z","iopub.status.idle":"2021-09-28T19:40:06.848006Z","shell.execute_reply.started":"2021-09-28T19:40:05.852854Z","shell.execute_reply":"2021-09-28T19:40:06.847073Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(5)","metadata":{"execution":{"iopub.status.busy":"2021-09-30T19:26:26.818644Z","iopub.execute_input":"2021-09-30T19:26:26.81892Z","iopub.status.idle":"2021-09-30T19:26:27.146382Z","shell.execute_reply.started":"2021-09-30T19:26:26.818895Z","shell.execute_reply":"2021-09-30T19:26:27.14389Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **DATA PRE-PROCESSING PART**","metadata":{}},{"cell_type":"code","source":"#remove words like ( is , been , have , ... )\ndef remove_stopwords(sentence):\n    words = sentence.split()\n    words = [word for word in words if word not in stopwords.words('english')]\n    \n    return ' '.join(words)\n\n#Using a regex, TO clean everything inside <>\ndef remove_html(sentence):\n    html = re.compile(r'<.*?>')\n    return html.sub(r'', sentence)\n\n#USE THE ABOVEV HELPER TO CLean question and long answer\ndef clean_df(df):\n    df['long_answer'] = df['long_answer'].apply(lambda x : remove_stopwords(x))\n    df['long_answer'] = df['long_answer'].apply(lambda x : remove_html(x))\n\n    df['question'] = df['question'].apply(lambda x : remove_stopwords(x))\n    df['question'] = df['question'].apply(lambda x : remove_html(x))\n    \n    return df","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:35:48.504226Z","iopub.execute_input":"2021-09-28T19:35:48.504543Z","iopub.status.idle":"2021-09-28T19:35:48.51296Z","shell.execute_reply.started":"2021-09-28T19:35:48.504517Z","shell.execute_reply":"2021-09-28T19:35:48.510361Z"}},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df = clean_df(train_df)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:35:50.400912Z","iopub.execute_input":"2021-09-28T19:35:50.401318Z","iopub.status.idle":"2021-09-28T19:37:28.52171Z","shell.execute_reply.started":"2021-09-28T19:35:50.401273Z","shell.execute_reply":"2021-09-28T19:37:28.520866Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Saving cleaned train data**","metadata":{}},{"cell_type":"code","source":"train_df.to_csv('mycsvfile.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2021-09-28T19:38:06.471229Z","iopub.execute_input":"2021-09-28T19:38:06.471559Z","iopub.status.idle":"2021-09-28T19:38:06.606239Z","shell.execute_reply.started":"2021-09-28T19:38:06.47153Z","shell.execute_reply":"2021-09-28T19:38:06.605159Z"}},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Reading cleaned train data**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\ntrain_df=pd.read_csv('../input/cleaned-tens/mycsvfile.csv')","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:00.502132Z","iopub.execute_input":"2021-10-07T21:22:00.502523Z","iopub.status.idle":"2021-10-07T21:22:00.619246Z","shell.execute_reply.started":"2021-10-07T21:22:00.502497Z","shell.execute_reply":"2021-10-07T21:22:00.618384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:00.622231Z","iopub.execute_input":"2021-10-07T21:22:00.622513Z","iopub.status.idle":"2021-10-07T21:22:00.636378Z","shell.execute_reply.started":"2021-10-07T21:22:00.622485Z","shell.execute_reply":"2021-10-07T21:22:00.635554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Tokenizer**","metadata":{}},{"cell_type":"code","source":"def define_tokenizer(df_series): #def of tokenizer engine \n    sentences = pd.concat(df_series)\n    \n    tokenizer =  tf.keras.preprocessing.text.Tokenizer(\n    num_words=None,\n    filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n    lower=True, split=' ', char_level=False, oov_token=None,\n    document_count=0) \n    \n   \n    tokenizer.fit_on_texts(sentences) #prepare text \n    \n    return tokenizer","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:00.64058Z","iopub.execute_input":"2021-10-07T21:22:00.640883Z","iopub.status.idle":"2021-10-07T21:22:00.647905Z","shell.execute_reply.started":"2021-10-07T21:22:00.640858Z","shell.execute_reply":"2021-10-07T21:22:00.647097Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = define_tokenizer([\n    train_df.long_answer, \n    train_df.question,\n    val_df.long_answer, \n    val_df.question\n])\ntokenizer.word_index['tracy']","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:00.651526Z","iopub.execute_input":"2021-10-07T21:22:00.651823Z","iopub.status.idle":"2021-10-07T21:22:01.24681Z","shell.execute_reply.started":"2021-10-07T21:22:00.651789Z","shell.execute_reply":"2021-10-07T21:22:01.245517Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Encoding with max lenght=300**","metadata":{}},{"cell_type":"code","source":"MAX_LEN=300\ndef encode(sentences, tokenizer):\n    encoded_sentences = tokenizer.texts_to_sequences(sentences)\n    encoded_sentences = tf.keras.preprocessing.sequence.pad_sequences(\n        encoded_sentences, \n        padding='post',\n        maxlen=MAX_LEN\n    )\n    return encoded_sentences","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:01.251305Z","iopub.execute_input":"2021-10-07T21:22:01.251758Z","iopub.status.idle":"2021-10-07T21:22:01.267097Z","shell.execute_reply.started":"2021-10-07T21:22:01.251688Z","shell.execute_reply":"2021-10-07T21:22:01.266076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_long_answers = encode(train_df['long_answer'].values, tokenizer)\ntrain_questions = encode(train_df['question'].values, tokenizer)\n\nval_long_answers = encode(val_df['long_answer'].values, tokenizer)\nval_questions = encode(val_df['question'].values, tokenizer)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:01.271093Z","iopub.execute_input":"2021-10-07T21:22:01.271497Z","iopub.status.idle":"2021-10-07T21:22:02.119151Z","shell.execute_reply.started":"2021-10-07T21:22:01.271456Z","shell.execute_reply":"2021-10-07T21:22:02.118307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" **Labels converted to 0-1 integers**","metadata":{}},{"cell_type":"code","source":"train_labels = train_df.is_long_answer.astype(int).values\nval_labels = val_df.is_long_answer.astype(int).values","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:02.121091Z","iopub.execute_input":"2021-10-07T21:22:02.121718Z","iopub.status.idle":"2021-10-07T21:22:02.12698Z","shell.execute_reply.started":"2021-10-07T21:22:02.12168Z","shell.execute_reply":"2021-10-07T21:22:02.126202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Import Words Embedding Using Glove**","metadata":{}},{"cell_type":"code","source":"embedding_dict = {}\n\nwith open('../input/glove-global-vectors-for-word-representation/glove.6B.' + str(EMBED_SIZE) + 'd.txt','r') as f:\n    for line in f:\n        values = line.split()\n        word = values[0]\n        vectors = np.asarray(values[1:],'float32')\n        embedding_dict[word] = vectors\n        \nf.close()","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:02.128303Z","iopub.execute_input":"2021-10-07T21:22:02.12868Z","iopub.status.idle":"2021-10-07T21:22:18.837114Z","shell.execute_reply.started":"2021-10-07T21:22:02.128645Z","shell.execute_reply":"2021-10-07T21:22:18.836237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Generate Embedding Matrix for our words ...**","metadata":{}},{"cell_type":"code","source":"num_words = len(tokenizer.word_index) + 1\nembedding_matrix = np.zeros((num_words, EMBED_SIZE))\n\nfor word, i in tokenizer.word_index.items():\n    if i > num_words:\n        continue\n    \n    emb_vec = embedding_dict.get(word)\n    \n    if emb_vec is not None:\n        embedding_matrix[i] = emb_vec","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:18.838519Z","iopub.execute_input":"2021-10-07T21:22:18.838917Z","iopub.status.idle":"2021-10-07T21:22:18.935013Z","shell.execute_reply.started":"2021-10-07T21:22:18.838879Z","shell.execute_reply":"2021-10-07T21:22:18.934155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **LONG MODEL PART**","metadata":{}},{"cell_type":"markdown","source":"**Embedding layer**","metadata":{}},{"cell_type":"code","source":"embedding = tf.keras.layers.Embedding(\n    len(tokenizer.word_index) + 1,\n    EMBED_SIZE,\n    embeddings_initializer = tf.keras.initializers.Constant(embedding_matrix),\n    trainable = False\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:18.93634Z","iopub.execute_input":"2021-10-07T21:22:18.936706Z","iopub.status.idle":"2021-10-07T21:22:18.955924Z","shell.execute_reply.started":"2021-10-07T21:22:18.936669Z","shell.execute_reply":"2021-10-07T21:22:18.955166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Architecture**","metadata":{}},{"cell_type":"code","source":"# question encoding\nquestion_input = Input(shape=(None,))\nquestion_x = embedding(question_input)\nquestion_x = SpatialDropout1D(0.2)(question_x)\nquestion_x = Bidirectional(LSTM(100, return_sequences=True))(question_x)\nquestion_x = GlobalMaxPooling1D()(question_x)\n\n# answer encoding\nanswer_input = Input(shape=(None,))\nanswer_x = embedding(answer_input)\nanswer_x = SpatialDropout1D(0.2)(answer_x)\nanswer_x = Bidirectional(LSTM(150, return_sequences=True))(answer_x)\nanswer_x = GlobalMaxPooling1D()(answer_x)\n\n# classification\ncombined_x = concatenate([question_x, answer_x])\ncombined_x = Dense(300, activation='relu')(combined_x)\ncombined_x = Dropout(0.5)(combined_x)\ncombined_x = Dense(300, activation='relu')(combined_x)\ncombined_x = Dropout(0.5)(combined_x)\noutput = Dense(1, activation='sigmoid')(combined_x)\n\n# combine model parts into one\nmodel = tf.keras.models.Model(inputs=[answer_input, question_input], outputs=output)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:18.957128Z","iopub.execute_input":"2021-10-07T21:22:18.957472Z","iopub.status.idle":"2021-10-07T21:22:21.784887Z","shell.execute_reply.started":"2021-10-07T21:22:18.957436Z","shell.execute_reply":"2021-10-07T21:22:21.784077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Compile**","metadata":{}},{"cell_type":"code","source":"model.compile(\n    loss='binary_crossentropy', \n    optimizer='adam',\n    metrics=['BinaryAccuracy' ,'Recall', 'Precision' ]\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:21.786198Z","iopub.execute_input":"2021-10-07T21:22:21.786521Z","iopub.status.idle":"2021-10-07T21:22:21.800885Z","shell.execute_reply.started":"2021-10-07T21:22:21.786487Z","shell.execute_reply":"2021-10-07T21:22:21.800003Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Callbacks**","metadata":{}},{"cell_type":"code","source":"callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=2, verbose=1),\n    tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1),\n]","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:21.803953Z","iopub.execute_input":"2021-10-07T21:22:21.804298Z","iopub.status.idle":"2021-10-07T21:22:21.810641Z","shell.execute_reply.started":"2021-10-07T21:22:21.804267Z","shell.execute_reply":"2021-10-07T21:22:21.809755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Train The Model**","metadata":{}},{"cell_type":"code","source":"history = model.fit(\n    x = [train_long_answers, train_questions], \n    y = train_labels,\n    validation_data = (\n        [val_long_answers, val_questions], \n        val_labels\n    ),\n    epochs = EPOCHS,\n    callbacks = callbacks,\n    class_weight = CLASS_WEIGHTS,\n    batch_size = BATCH_SIZE,\n    shuffle = True\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:22:21.812008Z","iopub.execute_input":"2021-10-07T21:22:21.812437Z","iopub.status.idle":"2021-10-07T21:43:02.303107Z","shell.execute_reply.started":"2021-10-07T21:22:21.812386Z","shell.execute_reply":"2021-10-07T21:43:02.302038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Save Model**","metadata":{}},{"cell_type":"code","source":"from keras.models import load_model\nmodel.save('./long_model_full')","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:02.304629Z","iopub.execute_input":"2021-10-07T21:43:02.304964Z","iopub.status.idle":"2021-10-07T21:43:47.895139Z","shell.execute_reply.started":"2021-10-07T21:43:02.304927Z","shell.execute_reply":"2021-10-07T21:43:47.894284Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Import our trained Model from Inputs**","metadata":{}},{"cell_type":"code","source":"from tensorflow import keras\nmodel = keras.models.load_model('../input/long-model-qa/long_model')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model Summary**","metadata":{}},{"cell_type":"code","source":"model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:48.002187Z","iopub.execute_input":"2021-10-07T21:43:48.002547Z","iopub.status.idle":"2021-10-07T21:43:48.016431Z","shell.execute_reply.started":"2021-10-07T21:43:48.002512Z","shell.execute_reply":"2021-10-07T21:43:48.014178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Loss & Accuarcy Visualization**","metadata":{}},{"cell_type":"code","source":"history=model.history","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:48.018378Z","iopub.execute_input":"2021-10-07T21:43:48.018745Z","iopub.status.idle":"2021-10-07T21:43:48.02262Z","shell.execute_reply.started":"2021-10-07T21:43:48.018708Z","shell.execute_reply":"2021-10-07T21:43:48.021764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"accuracy =history.history['binary_accuracy']\nval_accuracy = history.history['val_binary_accuracy']\nloss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs = range(len(accuracy))\nplt.plot(epochs, accuracy,  label='Training accuracy')\nplt.plot(epochs, val_accuracy,label='Validation accuracy')\nplt.title('Training and validation accuracy')\nplt.legend()\nplt.figure()\nplt.plot(epochs, loss, label='Training loss')\nplt.plot(epochs, val_loss,label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\nplt.show()\nprint('Num of Epochs: {0}'.format(\n    len(history.history['loss'])\n))","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:48.027634Z","iopub.execute_input":"2021-10-07T21:43:48.027879Z","iopub.status.idle":"2021-10-07T21:43:48.340343Z","shell.execute_reply.started":"2021-10-07T21:43:48.027851Z","shell.execute_reply":"2021-10-07T21:43:48.33928Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**F1 Score**","metadata":{}},{"cell_type":"code","source":"recall = history.history['recall'][-1]\nprecision = history.history['precision'][-1]\n\nprint('Train F1 score: {0:.4f}'.format(\n    2 * (precision * recall) / (precision + recall)\n))\n\nrecall = history.history['val_recall'][-1]\nprecision = history.history['val_precision'][-1]\n\nprint('Validation F1 score: {0:.4f}'.format(\n    2 * (precision * recall) / (precision + recall)\n))","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:48.342436Z","iopub.execute_input":"2021-10-07T21:43:48.342794Z","iopub.status.idle":"2021-10-07T21:43:48.349595Z","shell.execute_reply.started":"2021-10-07T21:43:48.342757Z","shell.execute_reply":"2021-10-07T21:43:48.348667Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **SHORT MODEL PART**","metadata":{}},{"cell_type":"markdown","source":"**Extract the short answer from a long answer.**","metadata":{}},{"cell_type":"code","source":"def get_short_answer(annotations, long_start, long_end):\n    if len(annotations['short_answers']) > 0:\n        short_start = annotations['short_answers'][0]['start_token']\n        short_end = annotations['short_answers'][0]['end_token']\n        \n        short_start = short_start - long_start\n        short_end = short_end - long_start\n        \n        return short_start, short_end\n    else:\n        return 0, 0\n    \n\ndef form_short_data_row(question, text, long_start, long_end, short_start, short_end):\n    long_answer = ' '.join(text[long_start:long_end])\n    short_answer = ' '.join(long_answer.split(' ')[short_start:short_end])\n    \n    row = {\n        'question': question,\n        'long_answer': long_answer,\n        'short_answer': short_answer,\n        'short_start': short_start,\n        'short_end': short_end\n    }\n    \n    return row\n\n\ndef load_short_data(file_path, questions_start, questions_end):\n    rows = []\n    \n    with open(file_path) as file:\n\n        for i in tqdm(range(questions_start, questions_end)):\n            line = get_line_of_data(file)\n            question, text, annotations = get_question_and_document(line)\n\n            for i, candidate in enumerate(line['long_answer_candidates']):\n                label, long_start, long_end = get_long_candidate(i, annotations, candidate)\n\n                if label == True:\n                    short_start, short_end = get_short_answer(annotations, long_start, long_end)\n                    \n                    rows.append(\n                        form_short_data_row(question, text, long_start, long_end, short_start, short_end)\n                    )\n        \n    return pd.DataFrame(rows)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:48.350909Z","iopub.execute_input":"2021-10-07T21:43:48.351409Z","iopub.status.idle":"2021-10-07T21:43:48.363418Z","shell.execute_reply.started":"2021-10-07T21:43:48.351358Z","shell.execute_reply":"2021-10-07T21:43:48.362589Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_short_df = load_short_data(TRAIN_PATH, 0, NUM_OF_TRAIN_QUESTIONS)\nval_short_df = load_short_data(TRAIN_PATH, NUM_OF_TRAIN_QUESTIONS, NUM_OF_VAL_QUESTIONS)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:48.364659Z","iopub.execute_input":"2021-10-07T21:43:48.365205Z","iopub.status.idle":"2021-10-07T21:43:49.731339Z","shell.execute_reply.started":"2021-10-07T21:43:48.365167Z","shell.execute_reply":"2021-10-07T21:43:49.730424Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_short_df.head()","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:49.732752Z","iopub.execute_input":"2021-10-07T21:43:49.733142Z","iopub.status.idle":"2021-10-07T21:43:49.745459Z","shell.execute_reply.started":"2021-10-07T21:43:49.733101Z","shell.execute_reply":"2021-10-07T21:43:49.744426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_long_answers = encode(train_short_df['long_answer'].values, tokenizer)\ntrain_questions = encode(train_short_df['question'].values, tokenizer)\n\nval_long_answers = encode(val_short_df['long_answer'].values, tokenizer)\nval_questions = encode(val_short_df['question'].values, tokenizer)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:49.746934Z","iopub.execute_input":"2021-10-07T21:43:49.747681Z","iopub.status.idle":"2021-10-07T21:43:49.883885Z","shell.execute_reply.started":"2021-10-07T21:43:49.747639Z","shell.execute_reply":"2021-10-07T21:43:49.883064Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def form_short_labels(df, sentence_length):\n    start_labels = np.zeros((len(df), sentence_length))\n    end_labels = np.zeros((len(df), sentence_length))\n\n    for i in range(len(df)):\n        start = df.loc[i].short_start\n        end = df.loc[i].short_end\n\n        if start < 300 and end < 300:\n            start_labels[i, start] = 1\n            end_labels[i, end] = 1\n        else:\n            continue\n    \n    return start_labels, end_labels\n\n\ntrain_start_labels, train_end_labels = form_short_labels(train_short_df, MAX_LEN)\nval_start_labels, val_end_labels = form_short_labels(val_short_df, MAX_LEN)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:49.885021Z","iopub.execute_input":"2021-10-07T21:43:49.885323Z","iopub.status.idle":"2021-10-07T21:43:50.039469Z","shell.execute_reply.started":"2021-10-07T21:43:49.88529Z","shell.execute_reply":"2021-10-07T21:43:50.038673Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# load from file\nembedding_dict = {}\n\nwith open('../input/glove-global-vectors-for-word-representation/glove.6B.' + str(SHORT_EMBED_SIZE) + 'd.txt','r') as f:\n    for line in f:\n        values = line.split()\n        word = values[0]\n        vectors = np.asarray(values[1:],'float32')\n        embedding_dict[word] = vectors\n        \nf.close()\n\n# write to matrix\nnum_words = len(tokenizer.word_index) + 1\nembedding_matrix = np.zeros((num_words, SHORT_EMBED_SIZE))\n\nfor word, i in tokenizer.word_index.items():\n    if i > num_words:\n        continue\n    \n    emb_vec = embedding_dict.get(word)\n    \n    if emb_vec is not None:\n        embedding_matrix[i] = emb_vec\n        \n# load as tensorflow embedding\nembedding = tf.keras.layers.Embedding(\n    len(tokenizer.word_index) + 1,\n    SHORT_EMBED_SIZE,\n    embeddings_initializer = tf.keras.initializers.Constant(embedding_matrix),\n    trainable = False\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:43:50.040641Z","iopub.execute_input":"2021-10-07T21:43:50.040996Z","iopub.status.idle":"2021-10-07T21:44:22.142945Z","shell.execute_reply.started":"2021-10-07T21:43:50.040957Z","shell.execute_reply":"2021-10-07T21:44:22.141993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Architecture**","metadata":{}},{"cell_type":"code","source":"# encode question\nquestion_input = Input(shape=(None,))\nquestion_x = embedding(question_input)\nquestion_x = SpatialDropout1D(0.2)(question_x)\nquestion_x = Bidirectional(LSTM(200, return_sequences=True))(question_x)\nquestion_x = Bidirectional(LSTM(100, return_sequences=True))(question_x)\n\n# encode answer\nanswer_input = Input(shape=(None,))\nanswer_x = embedding(answer_input)\nanswer_x = SpatialDropout1D(0.2)(answer_x)\nanswer_x = Bidirectional(LSTM(250, return_sequences=True))(answer_x)\nanswer_x = Bidirectional(LSTM(150, return_sequences=True))(answer_x)\n\n# merge the encodings\ncombined_x = concatenate([question_x, answer_x])\n\n# predict start index\nstart_x = Dropout(0.1)(combined_x) \nstart_x = Conv1D(1,1)(start_x)\nstart_x = Flatten()(start_x)\nstart_x = Activation('softmax', name='start_token_out')(start_x)\n\n# predict end index\nend_x = Dropout(0.1)(combined_x) \nend_x = Conv1D(1,1)(end_x)\nend_x = Flatten()(end_x)\nend_x = Activation('softmax', name='end_token_out')(end_x)\n\n# merge the parts into one model\nshort_model = tf.keras.models.Model(inputs=[answer_input, question_input], outputs=[start_x, end_x])","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:44:22.144176Z","iopub.execute_input":"2021-10-07T21:44:22.144524Z","iopub.status.idle":"2021-10-07T21:44:24.10649Z","shell.execute_reply.started":"2021-10-07T21:44:22.144491Z","shell.execute_reply":"2021-10-07T21:44:24.105623Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Compile**","metadata":{}},{"cell_type":"code","source":"short_model.compile(\n    loss='categorical_crossentropy', \n    optimizer='adam',\n    metrics=['categorical_accuracy', 'Recall', 'Precision']\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:44:24.107818Z","iopub.execute_input":"2021-10-07T21:44:24.10815Z","iopub.status.idle":"2021-10-07T21:44:24.120427Z","shell.execute_reply.started":"2021-10-07T21:44:24.108115Z","shell.execute_reply":"2021-10-07T21:44:24.119595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Callbacks**","metadata":{}},{"cell_type":"code","source":"callbacks = [\n    tf.keras.callbacks.ReduceLROnPlateau(monitor='loss', patience=3, verbose=1),\n    tf.keras.callbacks.EarlyStopping(monitor='loss', patience=5, verbose=1),\n]","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:44:24.123426Z","iopub.execute_input":"2021-10-07T21:44:24.123754Z","iopub.status.idle":"2021-10-07T21:44:24.130035Z","shell.execute_reply.started":"2021-10-07T21:44:24.123713Z","shell.execute_reply":"2021-10-07T21:44:24.129283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Train The Model**\n","metadata":{}},{"cell_type":"code","source":"history = short_model.fit(\n    x = [train_long_answers, train_questions], \n    y = [train_start_labels, train_end_labels],\n    validation_data = (\n        [val_long_answers, val_questions], \n        [val_start_labels, val_end_labels]\n    ),\n    epochs = SHORT_EPOCHS,\n    callbacks = callbacks,\n    batch_size = SHORT_BATCH_SIZE,\n    shuffle = True\n)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:44:24.131095Z","iopub.execute_input":"2021-10-07T21:44:24.131584Z","iopub.status.idle":"2021-10-07T21:51:44.446088Z","shell.execute_reply.started":"2021-10-07T21:44:24.131546Z","shell.execute_reply":"2021-10-07T21:51:44.445148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Save Model**\n","metadata":{}},{"cell_type":"markdown","source":"**Import our trained Model from Inputs**\n","metadata":{}},{"cell_type":"code","source":"# Recreate the exact same model, including weights and optimizer.\nshort_loaded_model = keras.models.load_model('../input/short-model-qa/short_model.h5')","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model Summary**","metadata":{}},{"cell_type":"code","source":"short_model.summary()","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:51:44.449268Z","iopub.execute_input":"2021-10-07T21:51:44.449556Z","iopub.status.idle":"2021-10-07T21:51:44.467682Z","shell.execute_reply.started":"2021-10-07T21:51:44.44953Z","shell.execute_reply":"2021-10-07T21:51:44.466923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Epoch: {0}'.format(len(history.history['loss'])))\nprint('Loss: {0}'.format(history.history['loss'][-1]))","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:51:44.470284Z","iopub.execute_input":"2021-10-07T21:51:44.470556Z","iopub.status.idle":"2021-10-07T21:51:44.4791Z","shell.execute_reply.started":"2021-10-07T21:51:44.470532Z","shell.execute_reply":"2021-10-07T21:51:44.478221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = history.history['loss']\nval_loss = history.history['val_loss']\nepochs = range(len(loss))\nplt.plot(epochs, loss, label='Training loss')\nplt.plot(epochs, val_loss,label='Validation loss')\nplt.title('Training and validation loss')\nplt.legend()\nplt.show()\nprint('Num of Epochs: {0}'.format(\n    len(history.history['loss'])\n))","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:51:44.482236Z","iopub.execute_input":"2021-10-07T21:51:44.482492Z","iopub.status.idle":"2021-10-07T21:51:44.645011Z","shell.execute_reply.started":"2021-10-07T21:51:44.482468Z","shell.execute_reply":"2021-10-07T21:51:44.644168Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## **TEST DATA PART**","metadata":{}},{"cell_type":"markdown","source":"**Load Test data**","metadata":{}},{"cell_type":"code","source":"def get_line_of_data_test(file):\n    line = file.readline()\n    line = json.loads(line)\n    \n    return line\n\n\ndef get_question_and_document_test(line):\n    example_id=line['example_id']\n    question = line['question_text']\n    text = line['document_text'].split(' ')\n    return question, text,example_id\n\ndef get_long_candidate_test(i,candidate):\n    # get place where long answer starts and ends in the document text\n    long_start = candidate['start_token']\n    long_end = candidate['end_token']\n    \n    return long_start, long_end\n\n\n#create dataset with two features (question and long_answer)\ndef form_data_row_test(question, text, long_start, long_end, example_id):\n    row = {\n        'question': question,\n        'long_answer': ' '.join(text[long_start:long_end]),\n        'long_start': long_start,\n        'long_end': long_end,\n        'example_id': example_id\n    }\n    return row\n\ndef load_test_data(file_path, questions_start, questions_end):\n    rows = []\n    candidates= []\n    \n    with open(file_path) as file:\n\n        for i in tqdm(range(questions_start, questions_end)):\n            line = get_line_of_data_test(file)\n            question, text, example_id = get_question_and_document_test(line)\n            for i, candidate in enumerate(line['long_answer_candidates']):\n                    long_start, long_end = get_long_candidate_test(i,candidate)\n                    rows.append(\n                        form_data_row_test(question, text, long_start, long_end, example_id)\n                    )\n            candidates.append(i+1)\n        \n    return pd.DataFrame(rows), candidates\n","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:51:44.646468Z","iopub.execute_input":"2021-10-07T21:51:44.647016Z","iopub.status.idle":"2021-10-07T21:51:44.657564Z","shell.execute_reply.started":"2021-10-07T21:51:44.646977Z","shell.execute_reply":"2021-10-07T21:51:44.656604Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Test_PATH = '../input/tensorflow2-question-answering/simplified-nq-test.jsonl'\n\ntest_df,test_candidates = load_test_data(Test_PATH, 0, 346)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:51:44.658807Z","iopub.execute_input":"2021-10-07T21:51:44.659159Z","iopub.status.idle":"2021-10-07T21:51:45.331938Z","shell.execute_reply.started":"2021-10-07T21:51:44.659123Z","shell.execute_reply":"2021-10-07T21:51:45.331072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:51:45.333227Z","iopub.execute_input":"2021-10-07T21:51:45.333762Z","iopub.status.idle":"2021-10-07T21:51:45.3478Z","shell.execute_reply.started":"2021-10-07T21:51:45.333721Z","shell.execute_reply":"2021-10-07T21:51:45.346661Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_question(question, long_answer):\n    sentences = [question, long_answer]\n    sentences = encode(sentences, tokenizer)\n    prediction = model.predict(\n        [np.expand_dims(sentences[1], axis=0), np.expand_dims(sentences[0], axis=0)]\n    )\n    return prediction","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:51:45.349361Z","iopub.execute_input":"2021-10-07T21:51:45.349763Z","iopub.status.idle":"2021-10-07T21:51:45.355576Z","shell.execute_reply.started":"2021-10-07T21:51:45.349728Z","shell.execute_reply":"2021-10-07T21:51:45.354464Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Predict Long Answers**","metadata":{}},{"cell_type":"code","source":"j=0                                           # iterator on the test candidates array \ncurr_test_cand=test_candidates[j]             # temp array for num of ansers per question \nprob_predictions=[]                           # temp array to store all answers probabilites per question\n\nlong_PredictionIdx=[]                    # Indces of the predicted long answer sample in the train set  \nlong_PredictionIdx.append(0)            # the first elemtn is reserverd for the offset of each question to the begninng of the test set\n\n\n#Final Answers , used to save in the submission file \nlong_prediction_ids=[]\nlong_prediction_strings=[]\n\n\n#*** The predict main loop ***#\nfor i in range(test_df.shape[0]):\n\n    curr_pred=test_question(test_df.question.iloc[i],test_df.long_answer.iloc[i])\n    \n    prob_predictions.append(curr_pred)\n    curr_test_cand-=1\n    #print(urr_test_cand,\"trrttrtr\")\n    # cuur anser was the last one\n    if(curr_test_cand==0):\n        \n        #find the max prob cadididate\n        long_pred = np.amax(prob_predictions)\n        long_pred_idx= np.argmax(prob_predictions)\n        offset=long_PredictionIdx[0]\n        idx = offset+long_pred_idx\n\n        \n        if(long_pred>=0.5):\n            long_Prediction_string=str(test_df.long_start.iloc[idx])+':'+str(test_df.long_end.iloc[idx])\n            long_prediction_strings.append(long_Prediction_string)\n                        \n        else:\n            long_Prediction_string=\"\"\n            long_prediction_strings.append(long_Prediction_string)\n            \n            \n        long_prediction_id =str(test_df.example_id.iloc[idx])+'_long,'\n        long_prediction_ids.append(long_prediction_id)\n        long_PredictionIdx.append(idx)\n        \n       #print(i,long_prediction_id,long_Prediction_string)\n        \n        #reset temp var to next question\n        j=j+1\n        long_PredictionIdx[0]=i+1\n        if j < len(test_candidates):\n            curr_test_cand=test_candidates[j]\n            prob_predictions=[]\n            \n#print(long_prediction_ids)\n#print(long_PredictionIdx)","metadata":{"execution":{"iopub.status.busy":"2021-10-07T21:51:45.357202Z","iopub.execute_input":"2021-10-07T21:51:45.357609Z","iopub.status.idle":"2021-10-07T22:12:52.76255Z","shell.execute_reply.started":"2021-10-07T21:51:45.35757Z","shell.execute_reply":"2021-10-07T22:12:52.759496Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Predict short Answer**","metadata":{}},{"cell_type":"code","source":"def test_short_answer(question, long_answer):\n    sentences = [long_answer, question]\n    sentences = encode(sentences, tokenizer)\n\n    predictions = short_model.predict(\n        [np.expand_dims(sentences[0], axis=0), np.expand_dims(sentences[1], axis=0)]\n    )\n    predictions = np.array(predictions)\n    pred_start = np.amax(predictions[0,0])\n    pred_end = np.amax(predictions[1,0])\n\n    return pred_start,pred_end\n","metadata":{"execution":{"iopub.status.busy":"2021-10-07T22:12:52.763767Z","iopub.status.idle":"2021-10-07T22:12:52.764487Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"short_example_ids=[]\nshort_prediction_strings=[]\nidx=[]\npredictions=[]\ni=0\n\nfor i in range(len(long_PredictionIdx)):\n    if(i==0):\n        continue\n        \n    idx = long_PredictionIdx[i]\n    #print(idx)\n    \n    start_prob,end_prob=test_short_answer(test_df.question.iloc[idx],test_df.long_answer.iloc[idx])  \n    predictions.append((start_prob, end_prob))\n\n\n    if(start_prob>=0.5 and end_prob>=0.5):\n        short_prediction_string=\"YES\"\n        short_prediction_strings.append(short_prediction_string)\n    else:\n        short_prediction_string=\"NO\"\n        short_prediction_strings.append(short_prediction_string)\n\n    short_example_id =str(test_df.example_id.iloc[idx])+'_short,'\n    short_example_ids.append(short_example_id)\n    \n    #print(i,short_example_id,short_prediction_string)    ","metadata":{"execution":{"iopub.status.busy":"2021-10-07T22:12:52.76579Z","iopub.status.idle":"2021-10-07T22:12:52.766336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Creating Submission File**","metadata":{}},{"cell_type":"code","source":"import csv \nheader = ['example_id', 'PredictionString']\nwith open(\"submission.csv\", \"w\") as csvfile:\n    writer = csv.writer(csvfile)\n    writer.writerow(header)\n    long_prediction_id=len(long_prediction_ids)\n\n    for value in  range(len(long_prediction_ids)):\n        long_prediction_id-=1 \n        writer.writerow([long_prediction_ids[value], long_prediction_strings[value]])\n        writer.writerow([short_example_ids[value], short_prediction_strings[value]])","metadata":{"execution":{"iopub.status.busy":"2021-10-07T22:12:52.767523Z","iopub.status.idle":"2021-10-07T22:12:52.768108Z"},"trusted":true},"execution_count":null,"outputs":[]}]}