{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\nimport os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n# path to input files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = []\nwith open('/kaggle/input/tensorflow2-question-answering/simplified-nq-train.jsonl', 'r') as file:\n    for i in range(100):\n        # read a sample of 100 json lines\n        train.append(json.loads(file.readline()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = train[0]\nsample.keys()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = []\nwith open('/kaggle/input/tensorflow2-question-answering/simplified-nq-test.jsonl', 'r') as file:\n    for i in range(100):\n        test.append(json.loads(file.readline()))\n\ntest[0].keys()\n\n# test set does not have annotations, but it has the long_answer_candidates","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample['annotations']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for s in train:\n    if len(s['annotations']) > 1:\n        print('Found more than 1')\n\n# didn't find any examples where there are more than 1 long_answer annotated","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for s in train:\n    if len(s['annotations'][0]['short_answers']) > 1:\n        print('Found more than 1')\n        \n# on the other hand, found a few examples where there are more than 1 short_answers annotated","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_long_answer(json_dict):\n    document_text = json_dict['document_text']\n    tokens = document_text.split(' ')\n    # even though annotatations is an array, it seems to be all length of 1\n    long_answer_key = json_dict['annotations'][0]['long_answer']\n    long_answer = tokens[long_answer_key['start_token']:long_answer_key['end_token']]\n    return \" \".join(long_answer)\n\nget_long_answer(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_short_answers(json_dict):\n    document_text = json_dict['document_text']\n    tokens = document_text.split(' ')\n    short_answer_keys = [short for short in json_dict['annotations'][0]['short_answers']]\n    short_answers = [\" \".join(tokens[key['start_token']:key['end_token']]) for key in short_answer_keys]\n    return short_answers\n\nget_short_answers(sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_long_short(train, i):\n    s = train[i]\n    print('question:')\n    print(s['question_text'])\n    print('')\n    \n    print('long answer:')\n    print(get_long_answer(s))\n    print('')\n    \n    print('short answer:')\n    print(\"\\n\".join(get_short_answers(s)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_long_short(train, 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_long_short(train, 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_long_short(train, 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_long_short(train, 90)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_long_short(train, 91)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print_long_short(train, 92)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}