{"cells":[{"metadata":{},"cell_type":"markdown","source":"Explore Yes/No Questions with Word2Vec"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import json\n\nimport numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\n\n\ndef jsonl_to_df(file_path, n_rows=1000, load_annotations=True, truncate=True, offset=200):\n    \"\"\"\n    Simple utility function to load the .jsonl files for the \n    TF2.0 QA competition. It creates a dataframe of the dataset.\n    \n    To use, click \"File\" > \"Add utility script\", search the name of this \n    notebook, then run:\n    \n    >>> from tf_qa_jsonl_to_dataframe import jsonl_to_df\n    >>> train = jsonl_to_df(\"/kaggle/...train.jsonl\")\n    >>> test = jsonl_to_df(\"/kaggle/...test.jsonl\", load_annotations=False)\n    \n    Parameters:\n        * file_path: The path to your json_file\n        * n: The number of rows you are importing\n        * truncate: Whether to cut the text before the first answer (long or short)\n          and after the last answer (long or short), leaving a space for the offset\n        * offset: If offset = k, then keep only keep the interval (answer_start - k, answer_end + k)\n        \n    Returns:\n        A Dataframe containing the following columns:\n            * document_text (str): The document split by whitespace, possibly truncated\n            * question_text (str): the question posed\n            * yes_no_answer (str): Could be \"YES\", \"NO\", or \"NONE\"\n            * short_answer_start (int): Start index of token, -1 if does not exist\n            * short_answer_start (int): End index of token, -1 if does not exist\n            * long_answer_start (int): Start index of token, -1 if does not exist\n            * long_answer_start (int): End index of token, -1 if does not exist\n        \n        And may contain:\n            * example_id (str): ID representing the string. Only for test data.\n    \n    Author: @xhlulu\n    Source: https://www.kaggle.com/xhlulu/tf-qa-jsonl-to-dataframe\n    \"\"\"\n    json_lines = []\n    \n    with open(file_path) as f:\n        for i, line in enumerate(tqdm(f)):\n            if not line:\n                break\n            if n_rows != -1 and i >= n_rows:\n                break\n                \n            line = json.loads(line)\n            last_token = line['long_answer_candidates'][-1]['end_token']\n\n            out_di = {\n                'document_text': line['document_text'],\n                'question_text': line['question_text']\n            }\n            \n            if 'example_id' in line:\n                out_di['example_id'] = line['example_id']\n            \n            if load_annotations:\n                annot = line['annotations'][0]\n                \n                out_di['yes_no_answer'] = annot['yes_no_answer']\n                out_di['long_answer_start'] = annot['long_answer']['start_token']\n                out_di['long_answer_end'] = annot['long_answer']['end_token']\n\n                if len(annot['short_answers']) > 0:\n                    out_di['short_answer_start'] = annot['short_answers'][0]['start_token']\n                    out_di['short_answer_end'] = annot['short_answers'][0]['end_token']\n                else:\n                    out_di['short_answer_start'] = -1\n                    out_di['short_answer_end'] = -1\n\n                if truncate:\n                    if out_di['long_answer_start'] == -1:\n                        start_threshold = out_di['short_answer_start'] - offset\n                    elif out_di['short_answer_start'] == -1:\n                        start_threshold = out_di['long_answer_start'] - offset\n                    else:\n                        start_threshold = min(out_di['long_answer_start'], out_di['short_answer_start']) - offset\n                        \n                    start_threshold = max(0, start_threshold)\n                    end_threshold = max(out_di['long_answer_end'], out_di['short_answer_end']) + offset + 1\n                    \n                    out_di['document_text'] = \" \".join(\n                        out_di['document_text'].split(' ')[start_threshold:end_threshold]\n                    )\n\n            json_lines.append(out_di)\n\n    df = pd.DataFrame(json_lines).fillna(-1)\n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if __name__ == '__main__':\n    directory = '/kaggle/input/tensorflow2-question-answering/'\n    train = jsonl_to_df(directory + 'simplified-nq-train.jsonl', n_rows = 200000)\n    test = jsonl_to_df(directory + 'simplified-nq-test.jsonl', n_rows = 1000, load_annotations=False)\n    print(train.shape)\n    print(test.shape)\n    \n    print(train.columns)\n    print(test.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.groupby(by=['yes_no_answer']).count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['question_text'].str.len().plot.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['yes_no_answer'] == 'YES']['question_text'].str.len().plot.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['yes_no_answer'] == 'NONE']['question_text'].str.len().plot.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train[train['yes_no_answer'] == 'NO']['question_text'].str.len().plot.hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_yes_questions = train[train['yes_no_answer'] == 'YES']['question_text']\ntrain_no_questions = train[train['yes_no_answer'] == 'NO']['question_text']\ntrain_none_questions = train[train['yes_no_answer'] == 'NONE']['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_corpus(data):\n    \"Creates a list of lists containing words from each yes/no questions\"\n    corpus = []\n    for sentence in data.iteritems():\n            word_list = sentence[1].split(\" \")\n            corpus.append(word_list)\n    return corpus","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import word2vec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.manifold import TSNE\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\ndef tsne_plot(model):\n    \"Creates and TSNE model and plots it\"\n    labels = []\n    tokens = []\n\n    for word in model.wv.vocab:\n        tokens.append(model[word])\n        labels.append(word)\n    \n    tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)\n    new_values = tsne_model.fit_transform(tokens)\n\n    x = []\n    y = []\n    for value in new_values:\n        x.append(value[0])\n        y.append(value[1])\n        \n    plt.figure(figsize=(16, 16)) \n    for i in range(len(x)):\n        plt.scatter(x[i],y[i])\n        plt.annotate(labels[i],\n                     xy=(x[i], y[i]),\n                     xytext=(5, 2),\n                     textcoords='offset points',\n                     ha='right',\n                     va='bottom')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corpus_yes = build_corpus(train_yes_questions)\nmodel_yes = word2vec.Word2Vec(corpus_yes, size=100, window=30, min_count=20)\ntsne_plot(model_yes)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corpus_no = build_corpus(train_no_questions)\nmodel_no = word2vec.Word2Vec(corpus_no, size=100, window=30, min_count=30)\ntsne_plot(model_no)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corpus_none = build_corpus(train_none_questions)\nmodel_none = word2vec.Word2Vec(corpus_none, size=100, window=250, min_count=1000)\ntsne_plot(model_none)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_yes.most_similar('is')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_no.most_similar('is')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_none.most_similar('is')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}