{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 导入工具包\nimport json\nimport numpy as np\nimport random\nimport torch\nfrom torch.utils.data import DataLoader, Dataset \nfrom transformers import AdamW, BertForQuestionAnswering, BertTokenizerFast\nfrom tqdm.auto import tqdm\n\n\n# Get device\ndevice = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# 固定随机种子以获得再现性\ndef same_seeds(seed):\n    torch.manual_seed(seed)\n    if torch.cuda.is_available():\n        torch.cuda.manual_seed(seed)\n        torch.cuda.manual_seed_all(seed)\n    np.random.seed(seed)\n    random.seed(seed)\n    torch.backends.cudnn.benchmark = False\n    torch.backends.cudnn.deterministic = True\nsame_seeds(1314)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-09T05:52:12.981584Z","iopub.execute_input":"2022-07-09T05:52:12.982251Z","iopub.status.idle":"2022-07-09T05:52:19.945048Z","shell.execute_reply.started":"2022-07-09T05:52:12.982161Z","shell.execute_reply":"2022-07-09T05:52:19.944179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 将\"fp16_training\"改为True， fp16是支持自动混合精度训练的，能够缩短模型的训练时间\nfp16_training = True\n\nif fp16_training:\n    !pip install accelerate==0.2.0\n    from accelerate import Accelerator\n    accelerator = Accelerator(fp16=True)\n    device = accelerator.device\n","metadata":{"execution":{"iopub.status.busy":"2022-07-09T05:52:19.946614Z","iopub.execute_input":"2022-07-09T05:52:19.947189Z","iopub.status.idle":"2022-07-09T05:52:31.076187Z","shell.execute_reply.started":"2022-07-09T05:52:19.947160Z","shell.execute_reply":"2022-07-09T05:52:31.075138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 加载模型及Tokenizer","metadata":{}},{"cell_type":"code","source":"# 加载模型及tokenizer\n# model_name = \"bert-base-chinese\" ，这是一个基础的预训练模型，想要模型的预测准确率高，可以选择下面的模型\nmodel_name = \"luhua/chinese_pretrain_mrc_macbert_large\"  \nmodel = BertForQuestionAnswering.from_pretrained(model_name).to(device)\ntokenizer = BertTokenizerFast.from_pretrained(model_name)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T05:52:31.077865Z","iopub.execute_input":"2022-07-09T05:52:31.078221Z","iopub.status.idle":"2022-07-09T05:53:16.385044Z","shell.execute_reply.started":"2022-07-09T05:52:31.078177Z","shell.execute_reply":"2022-07-09T05:53:16.384283Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 读入数据，并对数据进行预处理","metadata":{}},{"cell_type":"code","source":"# read_data函数是对输入文件进行处理后返回相应的question数据和paragraph数据\ndef read_data(file):\n    with open(file, 'r', encoding=\"utf-8\") as reader:\n        data = json.load(reader)\n    return data[\"questions\"], data[\"paragraphs\"]\n\ntrain_questions, train_paragraphs = read_data(\"../input/ml2022spring-hw7/hw7_train.json\")\ndev_questions, dev_paragraphs = read_data(\"../input/ml2022spring-hw7/hw7_dev.json\")\ntest_questions, test_paragraphs = read_data(\"../input/ml2022spring-hw7/hw7_test.json\")","metadata":{"execution":{"iopub.status.busy":"2022-07-09T05:53:16.387010Z","iopub.execute_input":"2022-07-09T05:53:16.387380Z","iopub.status.idle":"2022-07-09T05:53:17.039839Z","shell.execute_reply.started":"2022-07-09T05:53:16.387344Z","shell.execute_reply":"2022-07-09T05:53:17.038983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 分别标记问题和段落\n# 将文本数据转换为数字数据\ntrain_questions_tokenized = tokenizer([train_question[\"question_text\"] for train_question in train_questions], add_special_tokens=False)\ndev_questions_tokenized = tokenizer([dev_question[\"question_text\"] for dev_question in dev_questions], add_special_tokens=False)\ntest_questions_tokenized = tokenizer([test_question[\"question_text\"] for test_question in test_questions], add_special_tokens=False) \n\ntrain_paragraphs_tokenized = tokenizer(train_paragraphs, add_special_tokens=False)\ndev_paragraphs_tokenized = tokenizer(dev_paragraphs, add_special_tokens=False)\ntest_paragraphs_tokenized = tokenizer(test_paragraphs, add_special_tokens=False)\n\n# print(dev_paragraphs_tokenized)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T05:53:17.041205Z","iopub.execute_input":"2022-07-09T05:53:17.041613Z","iopub.status.idle":"2022-07-09T05:53:29.739017Z","shell.execute_reply.started":"2022-07-09T05:53:17.041573Z","shell.execute_reply":"2022-07-09T05:53:29.738176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset 与 Dataloader","metadata":{}},{"cell_type":"code","source":"train_batch_size = 8\ndoc_stride = 100\n\nclass QA_Dataset(Dataset):\n    def __init__(self, split, questions, tokenized_questions, tokenized_paragraphs, doc_stride=doc_stride):\n        self.split = split\n        self.questions = questions\n        self.tokenized_questions = tokenized_questions\n        self.tokenized_paragraphs = tokenized_paragraphs\n        self.max_question_len = 40\n        self.max_paragraph_len = 350\n        \n        ##### TODO: Change value of doc_stride #####\n        self.doc_stride = doc_stride\n\n        # Input sequence length = [CLS] + question + [SEP] + paragraph + [SEP]\n        self.max_seq_len = 1 + self.max_question_len + 1 + self.max_paragraph_len + 1\n\n    def __len__(self):\n        return len(self.questions)\n\n    def __getitem__(self, idx):\n        question = self.questions[idx]\n        tokenized_question = self.tokenized_questions[idx]\n        tokenized_paragraph = self.tokenized_paragraphs[question[\"paragraph_id\"]]\n\n        ##### TODO: Preprocessing #####\n        # Hint: How to prevent model from learning something it should not learn\n\n        if self.split == \"train\":\n            # Convert answer's start/end positions in paragraph_text to start/end positions in tokenized_paragraph  \n            answer_start_token = tokenized_paragraph.char_to_token(question[\"answer_start\"])\n            answer_end_token = tokenized_paragraph.char_to_token(question[\"answer_end\"])\n\n            # A single window is obtained by slicing the portion of paragraph containing the answer\n            #mid = (answer_start_token + answer_end_token) // 2\n            #paragraph_start = max(0, min(mid - self.max_paragraph_len // 2, len(tokenized_paragraph) - self.max_paragraph_len))\n            #paragraph_end = paragraph_start + self.max_paragraph_len\n            \n            start_min = max(0, answer_end_token - self.max_paragraph_len + 1)\n            start_max = min(answer_start_token, len(tokenized_paragraph) - self.max_paragraph_len)\n            start_max = max(start_min, start_max)\n            paragraph_start = random.randint(start_min, start_max + 1)\n            paragraph_end = paragraph_start + self.max_paragraph_len\n            \n            # Slice question/paragraph and add special tokens (101: CLS, 102: SEP)\n            input_ids_question = [101] + tokenized_question.ids[:self.max_question_len] + [102] \n            input_ids_paragraph = tokenized_paragraph.ids[paragraph_start : paragraph_end] + [102]\t\t\n            \n            # Convert answer's start/end positions in tokenized_paragraph to start/end positions in the window  \n            answer_start_token += len(input_ids_question) - paragraph_start\n            answer_end_token += len(input_ids_question) - paragraph_start\n            \n            # Pad sequence and obtain inputs to model \n            input_ids, token_type_ids, attention_mask = self.padding(input_ids_question, input_ids_paragraph)\n            return torch.tensor(input_ids), torch.tensor(token_type_ids), torch.tensor(attention_mask), answer_start_token, answer_end_token\n\n        # Validation/Testing\n        else:\n            input_ids_list, token_type_ids_list, attention_mask_list = [], [], []\n            \n            # Paragraph is split into several windows, each with start positions separated by step \"doc_stride\"\n            for i in range(0, len(tokenized_paragraph), self.doc_stride):\n                \n                # Slice question/paragraph and add special tokens (101: CLS, 102: SEP)\n                input_ids_question = [101] + tokenized_question.ids[:self.max_question_len] + [102]\n                input_ids_paragraph = tokenized_paragraph.ids[i : i + self.max_paragraph_len] + [102]\n                \n                # Pad sequence and obtain inputs to model\n                input_ids, token_type_ids, attention_mask = self.padding(input_ids_question, input_ids_paragraph)\n                \n                input_ids_list.append(input_ids)\n                token_type_ids_list.append(token_type_ids)\n                attention_mask_list.append(attention_mask)\n            \n            return torch.tensor(input_ids_list), torch.tensor(token_type_ids_list), torch.tensor(attention_mask_list)\n\n    def padding(self, input_ids_question, input_ids_paragraph):\n        # Pad zeros if sequence length is shorter than max_seq_len\n        padding_len = self.max_seq_len - len(input_ids_question) - len(input_ids_paragraph)\n        # Indices of input sequence tokens in the vocabulary\n        input_ids = input_ids_question + input_ids_paragraph + [0] * padding_len\n        # Segment token indices to indicate first and second portions of the inputs. Indices are selected in [0, 1]\n        token_type_ids = [0] * len(input_ids_question) + [1] * len(input_ids_paragraph) + [0] * padding_len\n        # Mask to avoid performing attention on padding token indices. Mask values selected in [0, 1]\n        attention_mask = [1] * (len(input_ids_question) + len(input_ids_paragraph)) + [0] * padding_len\n        \n        return input_ids, token_type_ids, attention_mask\n\n\n\ntrain_set = QA_Dataset(\"train\", train_questions, train_questions_tokenized, train_paragraphs_tokenized)\ndev_set = QA_Dataset(\"dev\", dev_questions, dev_questions_tokenized, dev_paragraphs_tokenized)\ntest_set = QA_Dataset(\"test\", test_questions, test_questions_tokenized, test_paragraphs_tokenized)\n\n\n# Note: Do NOT change batch size of dev_loader / test_loader !\n# Although batch size=1, it is actually a batch consisting of several windows from the same QA pair\ntrain_loader = DataLoader(train_set, batch_size=train_batch_size, shuffle=True, pin_memory=True)\ndev_loader = DataLoader(dev_set, batch_size=1, shuffle=False, pin_memory=True)\ntest_loader = DataLoader(test_set, batch_size=1, shuffle=False, pin_memory=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-09T05:53:29.740236Z","iopub.execute_input":"2022-07-09T05:53:29.740614Z","iopub.status.idle":"2022-07-09T05:53:29.764541Z","shell.execute_reply.started":"2022-07-09T05:53:29.740580Z","shell.execute_reply":"2022-07-09T05:53:29.763806Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Function for Evaluation","metadata":{}},{"cell_type":"code","source":"def evaluate(data, output, doc_stride=doc_stride, token_type_ids=None, paragraph=None, paragraph_tokenized=None):\n    ##### TODO: Postprocessing #####\n    # There is a bug and room for improvement in postprocessing \n    # Hint: Open your prediction file to see what is wrong \n    \n    answer = ''\n    max_prob = float('-inf')\n    num_of_windows = data[0].shape[1]\n    \n    for k in range(num_of_windows):\n        # Obtain answer by choosing the most probable start position / end position\n        start_prob, start_index = torch.max(output.start_logits[k], dim=0)\n        end_prob, end_index = torch.max(output.end_logits[k], dim=0)\n        \n        token_type_id = data[1][0][k].detach().cpu().numpy()\n        #[CLS] + [question] + [SEP] + [paragraph] + [SEP]\n        paragraph_start = token_type_id.argmax()\n        paragraph_end = len(token_type_id) - 1 - token_type_id[::-1].argmax() - 1\n        \n        if start_index > end_index or start_index < paragraph_start or end_index > paragraph_end:\n            continue\n        \n        # Probability of answer is calculated as sum of start_prob and end_prob\n        prob = start_prob + end_prob\n        \n        # Replace answer if calculated probability is larger than previous windows\n        if prob > max_prob:\n            # Convert tokens to chars (e.g. [1920, 7032] --> \"大 金\")\n            max_prob = prob\n            answer = tokenizer.decode(data[0][0][k][start_index : end_index + 1])\n            # 找到tokenized paragraph中对应的位置\n            origin_start = start_index + k * doc_stride - paragraph_start\n            origin_end = end_index + k * doc_stride - paragraph_start;\n            \n    # Remove spaces in answer (e.g. \"大 金\" --> \"大金\")    \n    answer = answer.replace(' ', '')\n    if '[UNK]' in answer:\n        print('发现 [UNK]，这表明有文字无法编码, 使用原始文本')\n        #print(\"Paragraph:\", paragraph)\n        #print(\"Paragraph:\", paragraph_tokenized.tokens)\n        print('--直接解码预测:', answer)\n        #找到原始文本中对应的位置\n        raw_start =  paragraph_tokenized.token_to_chars(origin_start)[0]\n        raw_end = paragraph_tokenized.token_to_chars(origin_end)[1]\n        answer = paragraph[raw_start:raw_end]\n        print('--原始文本预测:',answer)\n    \n   \n    return answer\n","metadata":{"execution":{"iopub.status.busy":"2022-07-09T05:53:29.767732Z","iopub.execute_input":"2022-07-09T05:53:29.768587Z","iopub.status.idle":"2022-07-09T05:53:29.780607Z","shell.execute_reply.started":"2022-07-09T05:53:29.768459Z","shell.execute_reply":"2022-07-09T05:53:29.779786Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"num_epoch = 1\nvalidation = True\nlogging_step = 100\ntotal_steps = num_epoch * len(train_loader)\nacc_steps = 1\nlearning_rate = 1e-5\noptimizer = AdamW(model.parameters(), lr=learning_rate)\nfrom transformers import get_linear_schedule_with_warmup\nscheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=100, num_training_steps=total_steps//acc_steps)\n\nif fp16_training:\n    model, optimizer, train_loader = accelerator.prepare(model, optimizer, train_loader) \n\nmodel.train()\n\nprint(\"Start Training ...\")\n\nif not validation:\n    dev_set = QA_Dataset(\"train\", dev_questions, dev_questions_tokenized, dev_paragraphs_tokenized)\n    train_set = torch.utils.data.ConcatDataset([train_set, dev_set])\n    train_loader = DataLoader(train_set, batch_size=train_batch_size, shuffle=True, pin_memory=True)\n\nfor epoch in range(num_epoch):\n    step = 1\n    train_loss = train_acc = 0\n    # optimizer.zero_grad()  # 基础版的在step+=1上面\n    for data in tqdm(train_loader):\t\n        # Load all data into GPU\n        data = [i.to(device) for i in data]\n        \n        # Model inputs: input_ids, token_type_ids, attention_mask, start_positions, end_positions (Note: only \"input_ids\" is mandatory)\n        # Model outputs: start_logits, end_logits, loss (return when start_positions/end_positions are provided)  \n        output = model(input_ids=data[0], token_type_ids=data[1], attention_mask=data[2], start_positions=data[3], end_positions=data[4])\n\n        # Choose the most probable start position / end position\n        start_index = torch.argmax(output.start_logits, dim=1)\n        end_index = torch.argmax(output.end_logits, dim=1)\n        \n        # Prediction is correct only if both start_index and end_index are correct\n        train_acc += ((start_index == data[3]) & (end_index == data[4])).float().mean()\n        train_loss += output.loss\n        \n        if fp16_training:\n            accelerator.backward(output.loss)\n        else:\n            output.loss.backward()\n        \n\n        step += 1\n        if step % acc_steps == 0:\n            #grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=0.1)\n            optimizer.step()\n            optimizer.zero_grad() # 这里的和第一个for下面的optimizer重复了\n            scheduler.step()\n\n        ##### TODO: Apply linear learning rate decay #####\n        \n        # Print training loss and accuracy over past logging step\n        if step % logging_step == 0:\n            lr = optimizer.state_dict()['param_groups'][0]['lr']  # 基础版的没有lr\n            print(f\"Epoch {epoch + 1} | Step {step} | loss = {train_loss.item() / logging_step:.3f}, acc = {train_acc / logging_step:.3f}, lr={lr}\")  # , lr={lr}\n            train_loss = train_acc = 0\n    \n    \n    print(\"now begining evaluating\")\n    # model.train()  # 基础版的是和with对齐的\n    if validation:  # 基础版的if是和for data in tqdm(train_loader)对齐的\n        print(\"Evaluating Dev Set ...\")\n        model.eval()\n        with torch.no_grad():\n            dev_acc = 0\n            for i, data in enumerate(tqdm(dev_loader)):\n                output = model(input_ids=data[0].squeeze(dim=0).to(device), token_type_ids=data[1].squeeze(dim=0).to(device), attention_mask=data[2].squeeze(dim=0).to(device))\n                    # prediction is correct only if answer text exactly matches\n                answer = evaluate(data, output, doc_stride=doc_stride,paragraph=dev_paragraphs[dev_questions[i][\"paragraph_id\"]], paragraph_tokenized=dev_paragraphs_tokenized[dev_questions[i][\"paragraph_id\"]])\n                    # 基础版的没有answer，evaluate函数只有data和output，后面的参数都没有\n                dev_acc +=  answer == dev_questions[i][\"answer_text\"]\n            print(f\"Validation | Epoch {epoch + 1} | acc = {dev_acc / len(dev_loader):.3f}\")\n        model.train()  # 基础版的是和with对齐的\n\n# Save a model and its configuration file to the directory 「saved_model」 \n# i.e. there are two files under the direcory 「saved_model」: 「pytorch_model.bin」 and 「config.json」\n# Saved model can be re-loaded using 「model = BertForQuestionAnswering.from_pretrained(\"saved_model\")」\nprint(\"Saving Model ...\")\nmodel_save_dir = \"saved_model\" \nmodel.save_pretrained(model_save_dir)","metadata":{"execution":{"iopub.status.busy":"2022-07-09T05:53:29.798135Z","iopub.execute_input":"2022-07-09T05:53:29.798704Z","iopub.status.idle":"2022-07-09T07:10:50.131653Z","shell.execute_reply.started":"2022-07-09T05:53:29.798666Z","shell.execute_reply":"2022-07-09T07:10:50.130275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Testing / Evaluating","metadata":{}},{"cell_type":"code","source":"print(\"Evaluating Test Set ...\")\n\nresult = []\n\nmodel.eval()\nwith torch.no_grad():\n    for i, data in enumerate(tqdm(test_loader)):\n        output = model(input_ids=data[0].squeeze(dim=0).to(device), token_type_ids=data[1].squeeze(dim=0).to(device),\n                       attention_mask=data[2].squeeze(dim=0).to(device))\n        result.append(evaluate(data, output, doc_stride=doc_stride, paragraph=test_paragraphs[test_questions[i][\"paragraph_id\"]],\n                               paragraph_tokenized=test_paragraphs_tokenized[test_questions[i][\"paragraph_id\"]]))\n\nresult_file = \"result.csv\"\nwith open(result_file, 'w') as f:\t\n\t  f.write(\"ID,Answer\\n\")\n\t  for i, test_question in enumerate(test_questions):\n        # Replace commas in answers with empty strings (since csv is separated by comma)\n\t\t    f.write(f\"{test_question['id']},{result[i].replace(',','')}\\n\")\n\nprint(f\"Completed! Result is in {result_file}\")","metadata":{"execution":{"iopub.status.busy":"2022-07-09T07:10:50.132962Z","iopub.status.idle":"2022-07-09T07:10:50.133430Z","shell.execute_reply.started":"2022-07-09T07:10:50.133191Z","shell.execute_reply":"2022-07-09T07:10:50.133214Z"},"trusted":true},"execution_count":null,"outputs":[]}]}