{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-16T04:34:52.248919Z","iopub.execute_input":"2021-12-16T04:34:52.249264Z","iopub.status.idle":"2021-12-16T04:34:52.261259Z","shell.execute_reply.started":"2021-12-16T04:34:52.249229Z","shell.execute_reply":"2021-12-16T04:34:52.260416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"! pip install 'git+https://github.com/PyTorchLightning/lightning-flash.git#egg=lightning-flash[text]' -q","metadata":{"execution":{"iopub.status.busy":"2021-12-16T04:34:52.475408Z","iopub.execute_input":"2021-12-16T04:34:52.475740Z","iopub.status.idle":"2021-12-16T04:35:22.756955Z","shell.execute_reply.started":"2021-12-16T04:34:52.475692Z","shell.execute_reply":"2021-12-16T04:35:22.755824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from flash import Trainer\nfrom flash.text import QuestionAnsweringData, QuestionAnsweringTask\nfrom flash.text.question_answering.input import QuestionAnsweringInputBase, QuestionAnsweringDictionaryInput\nimport pandas as pd\nimport json\nfrom typing import Union","metadata":{"execution":{"iopub.status.busy":"2021-12-16T06:02:38.325604Z","iopub.execute_input":"2021-12-16T06:02:38.326087Z","iopub.status.idle":"2021-12-16T06:02:38.330538Z","shell.execute_reply.started":"2021-12-16T06:02:38.326041Z","shell.execute_reply":"2021-12-16T06:02:38.329619Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"records = []\nn_samples = 100\nwith open('/kaggle/input/tensorflow2-question-answering/simplified-nq-train.jsonl', 'rt') as stream:\n    for i in range(n_samples):\n        records.append(json.loads(stream.readline()))\ndf_train = pd.DataFrame(records)","metadata":{"execution":{"iopub.status.busy":"2021-12-16T05:55:27.277743Z","iopub.execute_input":"2021-12-16T05:55:27.278067Z","iopub.status.idle":"2021-12-16T05:55:27.324970Z","shell.execute_reply.started":"2021-12-16T05:55:27.278033Z","shell.execute_reply":"2021-12-16T05:55:27.323984Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_data(file_path, questions_start, questions_end, short_answer=True):\n    ids = []\n    titles = []\n    contexts = []\n    questions = []\n    answers = []\n    \n    with open(file_path) as file:\n        for i in range(questions_start, questions_end):\n            line = json.loads(file.readline())\n            result = process_data(line, short_answer)\n            if result:\n                id_, title, context, question, answer = result\n                if answer:\n                    ids.append(id_)\n                    titles.append(title)\n                    contexts.append(context)\n                    questions.append(question)\n                    answers.append(answer)\n                    \n    data = {\"id\": ids, \"title\": titles, \"context\": contexts, \"question\": questions, \"answer\": answers}\n    \n    return pd.DataFrame(data)\n                \ndef process_data(entry, short_answer=True):\n    question = entry['question_text']\n    text = entry['document_text'].split(' ')\n    annotations = entry['annotations'][0]\n    id_ = entry['example_id']\n\n    for i, candidate in enumerate(entry['long_answer_candidates']):\n        isThereIndex = True if i == annotations['long_answer']['candidate_index'] else False\n        long_start = candidate['start_token']\n        long_end = candidate['end_token']\n        if isThereIndex:\n            short_start = 0 \n            short_end = 0\n            if len(annotations['short_answers']) > 0:\n                short_start = annotations['short_answers'][0]['start_token']\n                short_end = annotations['short_answers'][0]['end_token']\n\n                short_start = short_start - long_start\n                short_end = short_end - long_start\n            long_answer = ' '.join(text[long_start:long_end])\n            short_answer = ' '.join(long_answer.split(' ')[short_start:short_end])\n            if short_answer:\n                return (id_, '', ' '.join(text), question, short_answer)\n            else:\n                return (id_, '', ' '.join(text), question, long_answer)","metadata":{"execution":{"iopub.status.busy":"2021-12-16T06:10:23.232447Z","iopub.execute_input":"2021-12-16T06:10:23.233337Z","iopub.status.idle":"2021-12-16T06:10:23.247078Z","shell.execute_reply.started":"2021-12-16T06:10:23.233296Z","shell.execute_reply":"2021-12-16T06:10:23.246237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_data('../input/tensorflow2-question-answering/simplified-nq-train.jsonl', 0, 5, short_answer=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-16T06:10:24.512366Z","iopub.execute_input":"2021-12-16T06:10:24.512865Z","iopub.status.idle":"2021-12-16T06:10:24.534108Z","shell.execute_reply.started":"2021-12-16T06:10:24.512832Z","shell.execute_reply":"2021-12-16T06:10:24.533308Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"load_data('../input/tensorflow2-question-answering/simplified-nq-train.jsonl', 0, 5, short_answer=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-16T06:10:38.089097Z","iopub.execute_input":"2021-12-16T06:10:38.089363Z","iopub.status.idle":"2021-12-16T06:10:38.112661Z","shell.execute_reply.started":"2021-12-16T06:10:38.089332Z","shell.execute_reply":"2021-12-16T06:10:38.111810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://github.com/PyTorchLightning/lightning-flash/blob/052ed5299ac08e0cf94fa5b1697d64a97bbbe06e/flash/text/question_answering/input.py#L292\n# SQUAD data","metadata":{"execution":{"iopub.status.busy":"2021-12-16T05:55:39.821498Z","iopub.execute_input":"2021-12-16T05:55:39.822319Z","iopub.status.idle":"2021-12-16T05:55:39.827589Z","shell.execute_reply.started":"2021-12-16T05:55:39.822254Z","shell.execute_reply":"2021-12-16T05:55:39.826050Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class QuestionAnsweringTFInput(QuestionAnsweringDictionaryInput):\n    def load_data(\n        self,\n        json_file,\n        max_source_length: int = 384,\n        max_target_length: int = 30,\n        padding: Union[str, bool] = \"max_length\",\n        question_column_name: str = \"question\",\n        context_column_name: str = \"context\",\n        answer_column_name: str = \"answer\",\n        doc_stride: int = 128,\n    ):\n        ids = []\n        titles = []\n        contexts = []\n        questions = []\n        answers = []\n        \n        with open(json_file) as stream:\n            line = json.loads(stream.readline())\n            result = process_data(line, short_answer=True)\n            if result:\n                id_, title, context, question, answer = result\n                if answer:\n                    ids.append(id_)\n                    titles.append(title)\n                    contexts.append(title)\n                    questions.append(question)\n                    answers.append(answer)\n\n        data = {\"id\": ids, \"title\": titles, \"context\": contexts, \"question\": questions, \"answer\": answers}\n\n        return super().load_data(\n            data,\n            max_source_length=max_source_length,\n            max_target_length=max_target_length,\n            padding=padding,\n            question_column_name=question_column_name,\n            context_column_name=context_column_name,\n            answer_column_name=answer_column_name,\n            doc_stride=doc_stride,\n        )","metadata":{"execution":{"iopub.status.busy":"2021-12-16T06:03:18.621614Z","iopub.execute_input":"2021-12-16T06:03:18.621913Z","iopub.status.idle":"2021-12-16T06:03:18.633554Z","shell.execute_reply.started":"2021-12-16T06:03:18.621881Z","shell.execute_reply":"2021-12-16T06:03:18.632261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}