{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import re\nimport os\nimport gc\nimport json\nimport pickle\nimport fasttext\nimport Levenshtein\nimport numpy as np \nimport pandas as pd\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom tqdm import tqdm_notebook as tqdm \nfrom Levenshtein import ratio as levenshtein_distance\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction import text\nfrom scipy import spatial\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as patches\nfrom plotly import tools, subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected = True)\nimport plotly.graph_objs as go\nimport plotly.express as px\npd.set_option('max_columns', 1000)\nfrom bokeh.models import Panel, Tabs\nfrom bokeh.io import output_notebook, show\nfrom bokeh.plotting import figure\nimport lightgbm as lgb\nimport plotly.figure_factory as ff\nfrom sklearn.model_selection import KFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom keras.preprocessing import text, sequence\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom tqdm import tqdm_notebook as tqdm\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.layers import Input, Dense, Embedding, SpatialDropout1D, concatenate, Masking\nfrom tensorflow.keras.layers import LSTM, Bidirectional, GlobalMaxPooling1D, Dropout\nfrom tensorflow.keras.preprocessing import text, sequence\nfrom multiprocessing import Pool\nprint (\" Data is imported to squeeze and play\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Building the Train Data Set**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_train(train_path, n_rows=200000, sampling_rate=15):\n    with open(train_path) as f:\n        processed_rows = []\n\n        for i in tqdm(range(n_rows)):\n            line = f.readline()\n            if not line:\n                break\n\n            line = json.loads(line)\n\n            text = line['document_text'].split(' ')\n            question = line['question_text']\n            annotations = line['annotations'][0]\n\n            for i, candidate in enumerate(line['long_answer_candidates']):\n                label = i == annotations['long_answer']['candidate_index']\n\n                start = candidate['start_token']\n                end = candidate['end_token']\n\n                if label or (i % sampling_rate == 0):\n                    processed_rows.append({\n                        'text': \" \".join(text[start:end]),\n                        'is_long_answer': label,\n                        'question': question,\n                        'annotation_id': annotations['annotation_id']\n                    })\n\n        train = pd.DataFrame(processed_rows)\n        \n        return train\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Building the Test Data Set**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_test(test_path):\n    with open(test_path) as f:\n        processed_rows = []\n\n        for line in tqdm(f):\n            line = json.loads(line)\n\n            text = line['document_text'].split(' ')\n            question = line['question_text']\n            example_id = line['example_id']\n\n            for candidate in line['long_answer_candidates']:\n                start = candidate['start_token']\n                end = candidate['end_token']\n\n                processed_rows.append({\n                    'text': \" \".join(text[start:end]),\n                    'question': question,\n                    'example_id': example_id,\n                    'sequence': f'{start}:{end}'\n\n                })\n\n        test = pd.DataFrame(processed_rows)\n    \n    return test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"directory = '/kaggle/input/tensorflow2-question-answering/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_path = directory + 'simplified-nq-train.jsonl'\ntest_path = directory + 'simplified-nq-test.jsonl'\n\ntrain = build_train(train_path)\ntest = build_test(test_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### **Preprocessing**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def compute_text_and_questions(train, test, tokenizer):\n    train_text = tokenizer.texts_to_sequences(train.text.values)\n    train_questions = tokenizer.texts_to_sequences(train.question.values)\n    test_text = tokenizer.texts_to_sequences(test.text.values)\n    test_questions = tokenizer.texts_to_sequences(test.question.values)\n    \n    train_text = sequence.pad_sequences(train_text, maxlen=300)\n    train_questions = sequence.pad_sequences(train_questions)\n    test_text = sequence.pad_sequences(test_text, maxlen=300)\n    test_questions = sequence.pad_sequences(test_questions)\n    \n    return train_text, train_questions, test_text, test_questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = text.Tokenizer(lower=False, num_words=80000)\n\nfor text in tqdm([train.text, test.text, train.question, test.question]):\n    tokenizer.fit_on_texts(text.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_target = train.is_long_answer.astype(int).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_text, train_questions, test_text, test_questions = compute_text_and_questions(train, test, tokenizer)\ndel train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's Create the Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_embedding_matrix(tokenizer, path):\n    embedding_matrix = np.zeros((tokenizer.num_words + 1, 300))\n    ft_model = fasttext.load_model(path)\n\n    for word, i in tokenizer.word_index.items():\n        if i >= tokenizer.num_words - 1:\n            break\n        embedding_matrix[i] = ft_model.get_word_vector(word)\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(embedding_matrix):\n    embedding = Embedding(\n        *embedding_matrix.shape, \n        weights=[embedding_matrix], \n        trainable=False, \n        mask_zero=True\n    )\n    \n    q_in = Input(shape=(None,))\n    q = embedding(q_in)\n    q = SpatialDropout1D(0.2)(q)\n    q = Bidirectional(LSTM(100, return_sequences=True))(q)\n    q = GlobalMaxPooling1D()(q)\n    \n    \n    t_in = Input(shape=(None,))\n    t = embedding(t_in)\n    t = SpatialDropout1D(0.2)(t)\n    t = Bidirectional(LSTM(150, return_sequences=True))(t)\n    t = GlobalMaxPooling1D()(t)\n    \n    hidden = concatenate([q, t])\n    hidden = Dense(300, activation='relu')(hidden)\n    hidden = Dropout(0.5)(hidden)\n    hidden = Dense(300, activation='relu')(hidden)\n    hidden = Dropout(0.5)(hidden)\n    \n    out1 = Dense(1, activation='sigmoid')(hidden)\n    \n    model = Model(inputs=[t_in, q_in], outputs=out1)\n    model.compile(loss='binary_crossentropy', optimizer='adam')\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from pathlib import Path\nPATH = Path('/kaggle/input/tensorflow2-question-answering')\n!ls -1 {PATH}\nTEST_TOTAL = 346","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_joined_tokens(answer: dict) -> str:\n    return '%d:%d' % (answer['start_token'], answer['end_token'])\n\ndef get_pred(json_data: dict) -> dict:\n    ret = {'short': 'YES', 'long': ''}\n    candidates = json_data['long_answer_candidates']\n    \n    paragraphs = []\n    tokens = json_data['document_text'].split(' ')\n    for cand in candidates:\n        start_token = tokens[cand['start_token']]\n        if start_token == '<P>' and cand['top_level'] and cand['end_token']-cand['start_token']>35:\n            break\n    else:\n        cand = candidates[0]\n        \n    ret['long'] = get_joined_tokens(cand)\n    \n    id_ = str(json_data['example_id'])\n    ret = {id_+'_'+k: v for k, v in ret.items()} \n    return ret\n\npreds = dict()\n\nwith open(PATH / 'simplified-nq-test.jsonl', 'r') as f:\n    for line in tqdm(f, total=TEST_TOTAL):\n        json_data = json.loads(line) \n        prediction = get_pred(json_data)\n        preds.update(prediction)\n            \nsubmission = pd.read_csv(PATH / 'sample_submission.csv')\nsubmission['PredictionString'] = submission['example_id'].map(lambda x: preds[x])\nsubmission.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"},"widgets":{"application/vnd.jupyter.widget-state+json":{"state":{"0276c1ce501649d095f8aba415ded07b":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"109203c81bb6445c819410956965acf2":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_c87bed6ca03f495aa6632c4aec66bd34","placeholder":"​","style":"IPY_MODEL_af066864fd87459fb3015fdbba0adc0b","value":" 497/? [00:40&lt;00:00, 13.28it/s]"}},"37a987a3e52649b4a6cd50aa82d4f36b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_a80251082d7e4917ad1c6adf24b40967","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_af5568f5728b48b3833da1469cdd49a4","value":1}},"4348ff8c69bb4b459b767d3a5281aa5b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HTMLModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_0276c1ce501649d095f8aba415ded07b","placeholder":"​","style":"IPY_MODEL_58247006e53b43c8a8a3800f7750db29","value":" 346/? [00:19&lt;00:00, 18.21it/s]"}},"4d459526d08d4fd1b1908cc76ed2e8e0":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_d6714aadc9a24d6c8b052f8f4adfd7df","IPY_MODEL_109203c81bb6445c819410956965acf2"],"layout":"IPY_MODEL_87c7ad72024447359d5a3fa0779d08dc"}},"51a7a984bd0e4d1db712e12ccf067dec":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"58247006e53b43c8a8a3800f7750db29":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"6fcdc8fa449744a59d4eb090ee804ee2":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"85356d333c07417586748ae127ee6370":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"HBoxModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_37a987a3e52649b4a6cd50aa82d4f36b","IPY_MODEL_4348ff8c69bb4b459b767d3a5281aa5b"],"layout":"IPY_MODEL_6fcdc8fa449744a59d4eb090ee804ee2"}},"87c7ad72024447359d5a3fa0779d08dc":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a80251082d7e4917ad1c6adf24b40967":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"a8b57439c39d4444802bdc61c8f92ce3":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"af066864fd87459fb3015fdbba0adc0b":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"DescriptionStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"af5568f5728b48b3833da1469cdd49a4":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"ProgressStyleModel","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":"initial"}},"c87bed6ca03f495aa6632c4aec66bd34":{"model_module":"@jupyter-widgets/base","model_module_version":"1.2.0","model_name":"LayoutModel","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"d6714aadc9a24d6c8b052f8f4adfd7df":{"model_module":"@jupyter-widgets/controls","model_module_version":"1.5.0","model_name":"IntProgressModel","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"IntProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"danger","description":"","description_tooltip":null,"layout":"IPY_MODEL_51a7a984bd0e4d1db712e12ccf067dec","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_a8b57439c39d4444802bdc61c8f92ce3","value":1}}},"version_major":2,"version_minor":0}}},"nbformat":4,"nbformat_minor":1}