{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# BERT - Bidirectional Embedding Representations from Transformers\n\n\n## Overview\n\n**BERT**, or **B**idirectional **E**mbedding **R**epresentations from **T**ransformers, is a new method of pre-training language representations which obtains state-of-the-art results on a wide array of Natural Language Processing (NLP) tasks. The academic paper can be found here: https://arxiv.org/abs/1810.04805.\n\nThis Colab demonstates using Colab to fine-tune sentence classification tasks built on top of pretrained BERT models and \nrun predictions on tuned model.","metadata":{}},{"cell_type":"markdown","source":"## Install dependences\n- pytorch-lightning: a simple trainer to help you minize code base\n- transformers: library contains multiple BERT models\n- sentencepiece: a word-to-vect library with fast implementation","metadata":{}},{"cell_type":"code","source":"!pip install -q pytorch-lightning\n!pip install -q transformers\n!pip install -q sentencepiece\n!pip install -q fairseq","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-08T10:40:14.697443Z","iopub.execute_input":"2022-01-08T10:40:14.697827Z","iopub.status.idle":"2022-01-08T10:40:54.697971Z","shell.execute_reply.started":"2022-01-08T10:40:14.697727Z","shell.execute_reply":"2022-01-08T10:40:54.696923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# include some dependence\nimport pandas as pd\nimport numpy as np\nfrom torch.utils.data import random_split, DataLoader, Dataset\nimport pytorch_lightning as pl\nimport torch.nn as nn\nimport torch\nfrom typing import Optional, Union\ntrain_ratio = 0.8\nDATA_DIR = '../input/quora-insincere-questions-classification/train.csv'\nDATA_DIR_TEST = '../input/quora-insincere-questions-classification/test.csv'","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:15.320459Z","iopub.execute_input":"2022-01-08T09:16:15.320747Z","iopub.status.idle":"2022-01-08T09:16:18.949856Z","shell.execute_reply.started":"2022-01-08T09:16:15.320715Z","shell.execute_reply":"2022-01-08T09:16:18.948666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LOAD DATA","metadata":{}},{"cell_type":"code","source":"# Use pandas to read csv, this will return a excel like table data\ntrain = pd.read_csv(DATA_DIR, index_col=0)\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:18.951427Z","iopub.execute_input":"2022-01-08T09:16:18.951675Z","iopub.status.idle":"2022-01-08T09:16:24.643172Z","shell.execute_reply.started":"2022-01-08T09:16:18.951646Z","shell.execute_reply":"2022-01-08T09:16:24.642373Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class SentimentData(Dataset):\n \"\"\"\n Dataset class for sentiment analysis. \n Every dataset using pytorch should be overwrite this class\n This require 2 function, __len__ and __getitem__\n \"\"\"\n def __init__(self, data_dir):\n \"\"\"\n Args:\n data_dir (string): Directory with the csv file\n \"\"\"\n self.df = pd.read_csv(data_dir, index_col=0)\n\n def __len__(self):\n \"\"\"\n length of the dataset, i.e. number of rows in the csv file\n Returns: int \n \"\"\"\n return len(self.df)\n\n def __getitem__(self, idx):\n \"\"\"\n given a row index, returns the corresponding row of the csv file\n Returns: text (string), label (int) \n \"\"\"\n text = self.df[\"question_text\"][idx]\n label = self.df[\"target\"][idx]\n\n return text, label\n\n\nclass SentimentDataModule(pl.LightningDataModule):\n \"\"\"\n Module class for sentiment analysis. this class is used to load the data to the model. \n It is a subclass of LightningDataModule. \n \"\"\"\n\n def __init__(self, data_dir: str = DATA_DIR, batch_size: int = 16):\n \"\"\"\n Args:\n data_dir (string): Directory with the csv file\n batch_size (int): batch size for dataloader\n \"\"\"\n super().__init__()\n self.data_dir = data_dir\n self.batch_size = batch_size\n\n def setup(self, stage: Optional[str] = None):\n \"\"\"\n Loads the data to the model. \n the data is loaded in the setup function, so that it is loaded only once. \n \"\"\"\n data_full = SentimentData(self.data_dir)\n train_size = round(len(data_full) * train_ratio)\n val_size = len(data_full) - train_size\n print(len(data_full), train_size, val_size)\n self.data_train, self.data_val = random_split(data_full, [train_size, val_size])\n\n def train_dataloader(self):\n \"\"\"\n Returns: dataloader for training\n \"\"\"\n return DataLoader(self.data_train, batch_size=self.batch_size)\n\n def val_dataloader(self):\n \"\"\"\n Returns: dataloader for validation\n \"\"\"\n return DataLoader(self.data_val, batch_size=self.batch_size)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:24.645885Z","iopub.execute_input":"2022-01-08T09:16:24.646234Z","iopub.status.idle":"2022-01-08T09:16:24.660911Z","shell.execute_reply.started":"2022-01-08T09:16:24.646187Z","shell.execute_reply":"2022-01-08T09:16:24.660250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"from fairseq.data import Dictionary\nimport sentencepiece as spm\nfrom os.path import join as pjoin\nfrom transformers import PreTrainedTokenizer\nimport sentencepiece as spm\n\n\nclass XLMRobertaTokenizer(PreTrainedTokenizer):\n \"\"\"\n XLM-RoBERTa tokenizer adapted from transformers.PreTrainedTokenizer. This helps to convert the input text into \n tokenized format. eg, \n \n input: \"Hello, how are you?\" output: [\"1\", \"2\", \"3\", \"65\", \"2\", \"1\"]\n \n this class also provides the method to convert the tokenized format into the original text.\n \n eg, input: [\"1\", \"2\", \"3\", \"65\", \"2\", \"1\"] output: \"Hello, how are you?\"\n \n \"\"\"\n def __init__(\n self,\n pretrained_file,\n bos_token=\"\",\n eos_token=\"\",\n sep_token=\"\",\n cls_token=\"\",\n unk_token=\"\",\n pad_token=\"\",\n mask_token=\"\",\n **kwargs\n ):\n \"\"\"\n :param pretrained_file: path to the pretrained model file\n :param bos_token: beginning of sentence token\n :param eos_token: end of sentence token\n :param sep_token: separation token\n :param cls_token: classification token\n :param unk_token: unknown token\n :param pad_token: padding token\n :param mask_token: mask token\n \"\"\"\n super().__init__(\n bos_token=bos_token,\n eos_token=eos_token,\n unk_token=unk_token,\n sep_token=sep_token,\n cls_token=cls_token,\n pad_token=pad_token,\n mask_token=mask_token,\n **kwargs,\n )\n # load bpe model and vocab file\n sentencepiece_model = pjoin(pretrained_file, 'sentencepiece.bpe.model')\n vocab_file = pjoin(pretrained_file, 'dict.txt')\n self.sp_model = spm.SentencePieceProcessor()\n self.sp_model.Load(\n sentencepiece_model) # please dont use anything from sp_model bcz it makes everything goes wrong\n self.bpe_dict = Dictionary().load(vocab_file)\n # Mimic fairseq token-to-id alignment for the first 4 token\n self.fairseq_tokens_to_ids = {\"\": 0, \"\": 1, \"\": 2, \"\": 3}\n # The first \"real\" token \",\" has position 4 in the original fairseq vocab and position 3 in the spm vocab\n self.fairseq_offset = 0\n self.fairseq_tokens_to_ids[\"\"] = len(self.bpe_dict) + self.fairseq_offset\n self.fairseq_ids_to_tokens = {v: k for k, v in self.fairseq_tokens_to_ids.items()}\n\n def _tokenize(self, text):\n \"\"\" Tokenize a string. \"\"\"\n return self.sp_model.EncodeAsPieces(text)\n\n def _convert_token_to_id(self, token):\n \"\"\" Converts a token (str) in an id using the vocab. \"\"\"\n if token in self.fairseq_tokens_to_ids:\n return self.fairseq_tokens_to_ids[token]\n spm_id = self.bpe_dict.index(token)\n return spm_id\n\n def _convert_id_to_token(self, index):\n \"\"\"Converts an index (integer) in a token (str) using the vocab.\"\"\"\n if index in self.fairseq_ids_to_tokens:\n return self.fairseq_ids_to_tokens[index]\n return self.bpe_dict[index]\n\n @property\n def vocab_size(self):\n \"\"\" Size of the base vocabulary (without the added tokens) \"\"\"\n return len(self.bpe_dict) + self.fairseq_offset + 1 # Add the token\n\n def get_vocab(self):\n \"\"\" Returns the vocabulary as a list of tokens. \"\"\"\n vocab = {self.convert_ids_to_tokens(i): i for i in range(self.vocab_size)}\n vocab.update(self.added_tokens_encoder)\n return vocab","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:24.662396Z","iopub.execute_input":"2022-01-08T09:16:24.662861Z","iopub.status.idle":"2022-01-08T09:16:25.113878Z","shell.execute_reply.started":"2022-01-08T09:16:24.662828Z","shell.execute_reply":"2022-01-08T09:16:25.112636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"from transformers import RobertaTokenizer, RobertaForSequenceClassification\nimport torch\n\npretrained_path = 'roberta-base'\n# !ls $pretrained_path\n# load tokenizer\nroberta = RobertaForSequenceClassification.from_pretrained(pretrained_path)\ntokenizer = RobertaTokenizer.from_pretrained(pretrained_path)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:25.115526Z","iopub.execute_input":"2022-01-08T09:16:25.115866Z","iopub.status.idle":"2022-01-08T09:16:52.115392Z","shell.execute_reply.started":"2022-01-08T09:16:25.115823Z","shell.execute_reply":"2022-01-08T09:16:52.114370Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inputs = tokenizer(\"Hello, my dog is cute\", return_tensors=\"pt\")\nlabels = torch.tensor([1]).unsqueeze(0) # Batch size 1\noutputs = roberta(**inputs, labels=labels)\nloss = outputs.loss\nlogits = outputs.logits\nprint(inputs)\nprint(logits)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:52.116962Z","iopub.execute_input":"2022-01-08T09:16:52.117296Z","iopub.status.idle":"2022-01-08T09:16:52.315710Z","shell.execute_reply.started":"2022-01-08T09:16:52.117251Z","shell.execute_reply":"2022-01-08T09:16:52.314876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# try to convert some text into numbers\ninputs = \"I love you\"\ninputs = tokenizer(inputs, return_tensors='pt')\nlabels=torch.tensor([0, 1, 1])\nlabels = torch.tensor([1]).unsqueeze(0)\nprint(inputs)\noutputs = roberta(**inputs, labels=labels)\nprint(outputs)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:52.316819Z","iopub.execute_input":"2022-01-08T09:16:52.317512Z","iopub.status.idle":"2022-01-08T09:16:52.374235Z","shell.execute_reply.started":"2022-01-08T09:16:52.317474Z","shell.execute_reply":"2022-01-08T09:16:52.373525Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import roc_auc_score, classification_report, accuracy_score\n\n\nclass SentimentRoberta(pl.LightningModule):\n \"\"\"\n SentimentRoberta class inherits from LightningModule\n This class is used to train a model using PyTorch Lightning\n It overrides the following methods:\n - forward : forward pass of the model\n - training_step : training step of the model\n - validation_step : validation step of the model\n - validation_epoch_end : end of the validation epoch\n - configure_optimizers : configure optimizers\n \"\"\"\n def __init__(self, lr_roberta, lr_classifier):\n \"\"\"\n Initialize the model with the following parameters:\n - lr_roberta : learning rate of the roberta model\n - lr_classifier : learning rate of the classifier model\n \"\"\"\n super().__init__()\n self.roberta = RobertaForSequenceClassification.from_pretrained(pretrained_path)\n self.tokenizer = RobertaTokenizer.from_pretrained(pretrained_path)\n self.lr_roberta = lr_roberta\n self.lr_classifer = lr_classifier\n\n def forward(self, texts, labels=None):\n \"\"\"\n Forward pass of the model\n Args:\n - texts : input texts\n - labels : labels of the input texts\n \"\"\"\n inputs = self.tokenizer(texts, return_tensors='pt', padding=True, truncation=True, max_length=256)\n for key in inputs:\n inputs[key] = inputs[key].to(self.device)\n outputs = self.roberta(**inputs, labels=labels)\n return outputs\n\n def configure_optimizers(self):\n \"\"\"\n Configure optimizers\n This method is used to configure the optimizers of the model by using the learning rate\n for specific parameter of the roberta model and the classifier model\n \"\"\"\n roberta_params = self.roberta.roberta.named_parameters()\n classifier_params = self.roberta.classifier.named_parameters()\n\n grouped_params = [\n {\"params\": [p for n, p in roberta_params], \"lr\": self.lr_roberta},\n {\"params\": [p for n, p in classifier_params], \"lr\": self.lr_classifer}\n ]\n optimizer = torch.optim.AdamW(\n grouped_params\n )\n scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=1000, gamma=0.98)\n return {\n 'optimizer': optimizer,\n 'lr_scheduler': {\n 'scheduler': scheduler,\n 'monitor': 'f1/val',\n }\n }\n\n def training_step(self, batch, batch_idx):\n \"\"\"\n Training step of the model\n Args:\n - batch : batch of the data\n - batch_idx : index of the batch\n \"\"\"\n texts, labels = batch\n outputs = self(texts, labels=labels)\n\n if len(outputs.values()) == 3:\n loss, logits, _ = outputs.values()\n else:\n loss, logits = outputs.values()\n return loss\n\n def validation_step(self, batch, batch_idx):\n \"\"\"\n Validation step of the model, used to compute the metrics\n Args:\n - batch : batch of the data\n - batch_idx : index of the batch\n \"\"\"\n texts, labels = batch\n outputs = self(texts, labels=labels)\n\n if len(outputs.values()) == 3:\n loss, logits, _ = outputs.values()\n else:\n loss, logits = outputs.values()\n\n output_scores = torch.softmax(logits, dim=-1)\n return loss, output_scores, labels\n\n def validation_epoch_end(self, validation_step_outputs):\n \"\"\"\n End of the validation epoch, this method will be called at the end of the validation epoch,\n it will compute the multiple metrics of classification problem\n Args:\n - validation_step_outputs : outputs of the validation step\n \"\"\"\n\n val_preds = torch.tensor([], device=self.device)\n val_scores = torch.tensor([], device=self.device)\n val_labels = torch.tensor([], device=self.device)\n val_loss = 0\n total_item = 0\n\n for idx, item in enumerate(validation_step_outputs):\n loss, output_scores, labels = item\n\n predictions = torch.argmax(output_scores, dim=-1)\n val_preds = torch.cat((val_preds, predictions), dim=0)\n val_scores = torch.cat((val_scores, output_scores[:, 1]), dim=0)\n val_labels = torch.cat((val_labels, labels), dim=0)\n\n val_loss += loss\n total_item += 1\n\n # print(\"VAL PREDS\", val_preds.shape)\n # print(\"VAL SCORES\", val_scores.shape)\n # print(\"VAL LABELS\", val_labels.shape)\n val_preds = val_preds.cpu().numpy()\n val_scores = val_scores.cpu().numpy()\n val_labels = val_labels.cpu().numpy()\n\n reports = classification_report(val_labels, val_preds, output_dict=True)\n print(\"VAL LABELS\", val_labels)\n print(\"VAL SCORES\", val_scores)\n try:\n auc = roc_auc_score(val_labels, val_scores)\n except Exception as e:\n print(e)\n print(\"Cannot calculate AUC. Default to 0\")\n auc = 0\n accuracy = accuracy_score(val_labels, val_preds)\n\n print(classification_report(val_labels, val_preds))\n\n self.log(\"loss/val\", val_loss)\n self.log(\"auc/val\", auc)\n self.log(\"accuracy/val\", accuracy)\n self.log(\"precision/val\", reports[\"weighted avg\"][\"precision\"])\n self.log(\"recall/val\", reports[\"weighted avg\"][\"recall\"])\n self.log(\"f1/val\", reports[\"weighted avg\"][\"f1-score\"])","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:52.375789Z","iopub.execute_input":"2022-01-08T09:16:52.376255Z","iopub.status.idle":"2022-01-08T09:16:53.178803Z","shell.execute_reply.started":"2022-01-08T09:16:52.376221Z","shell.execute_reply":"2022-01-08T09:16:53.177968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntrainer = pl.Trainer(\n fast_dev_run=True,\n)\nmodel = SentimentRoberta(lr_roberta=1e-5, lr_classifier=3e-3)\ndm = SentimentDataModule(DATA_DIR)\n\ntrainer.fit(model, dm)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:16:53.181712Z","iopub.execute_input":"2022-01-08T09:16:53.182052Z","iopub.status.idle":"2022-01-08T09:17:06.547295Z","shell.execute_reply.started":"2022-01-08T09:16:53.182007Z","shell.execute_reply":"2022-01-08T09:17:06.546582Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir checkpoint\n!mkdir tensorboard","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:17:06.548506Z","iopub.execute_input":"2022-01-08T09:17:06.548849Z","iopub.status.idle":"2022-01-08T09:17:08.324660Z","shell.execute_reply.started":"2022-01-08T09:17:06.548819Z","shell.execute_reply":"2022-01-08T09:17:08.323418Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Training","metadata":{}},{"cell_type":"code","source":"from pytorch_lightning import loggers as pl_loggers\nfrom pytorch_lightning.callbacks import EarlyStopping, ModelCheckpoint\n\ntorch.manual_seed(123)\n\ntb_logger = pl_loggers.TensorBoardLogger('./tensorboard')\n\ntrainer = pl.Trainer(\n min_epochs=1,\n max_epochs=1,\n gpus=1,\n precision=16,\n val_check_interval=0.5,\n # check_val_every_n_epoch=1,\n callbacks=[\n ModelCheckpoint(\n dirpath='./checkpoint',\n save_top_k=3,\n monitor='f1/val',\n ), \n EarlyStopping('f1/val', patience=5)\n ],\n fast_dev_run=False,\n logger=tb_logger\n)\n\ndm.setup(stage=\"fit\")\ntrainer.fit(model, dm)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T09:17:08.326690Z","iopub.execute_input":"2022-01-08T09:17:08.327237Z","iopub.status.idle":"2022-01-08T09:17:08.850098Z","shell.execute_reply.started":"2022-01-08T09:17:08.327199Z","shell.execute_reply":"2022-01-08T09:17:08.848397Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# show the result here\n%reload_ext tensorboard\n%tensorboard --logdir './tensorboard'","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}}]}