{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"from transformers import AutoTokenizer\nfrom torch.utils.data import Dataset\nimport pandas as pd\nimport numpy as np\nimport warnings\nimport os\n\n\nwarnings.simplefilter(\"ignore\")\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-04-08T16:33:29.475078Z","iopub.execute_input":"2023-04-08T16:33:29.475510Z","iopub.status.idle":"2023-04-08T16:33:29.481340Z","shell.execute_reply.started":"2023-04-08T16:33:29.475473Z","shell.execute_reply":"2023-04-08T16:33:29.480217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class PretrainingDataset(Dataset):\n    def __init__(self, texts, tokenizer, texts_pair=None, max_length=512):\n        super().__init__()\n        \n        self.texts = texts\n        self.texts_pair = texts_pair\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n        \n        if self.texts_pair is not None:\n            assert len(self.texts) == len(self.texts_pair)\n        \n    def __len__(self):\n        return len(self.texts)\n    \n    def tokenize(self, text, text_pair=None):\n        return self.tokenizer(\n            text=text, \n            text_pair=text_pair,\n            max_length=self.max_length,\n            truncation=True,\n            padding=False, \n            return_attention_mask=True,\n            add_special_tokens=True,\n            return_special_tokens_mask=True,\n            return_token_type_ids=False,\n            return_offsets_mapping=False,\n            return_tensors=None,\n        )\n    \n    def __getitem__(self, index):\n        text = self.texts[index]\n        \n        text_pair = None\n        if self.texts_pair is not None:\n            text_pair = self.texts_pair[index]\n            \n        tokenized = self.tokenize(text)\n        \n        return tokenized","metadata":{"execution":{"iopub.status.busy":"2023-04-08T16:33:29.651969Z","iopub.execute_input":"2023-04-08T16:33:29.653124Z","iopub.status.idle":"2023-04-08T16:33:29.662200Z","shell.execute_reply.started":"2023-04-08T16:33:29.653082Z","shell.execute_reply":"2023-04-08T16:33:29.660940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"/kaggle/input/feedback-prize-english-language-learning/train.csv\"\ndata = pd.read_csv(data_path)\n\ntexts = data[\"full_text\"].values","metadata":{"execution":{"iopub.status.busy":"2023-04-08T16:33:29.785184Z","iopub.execute_input":"2023-04-08T16:33:29.785575Z","iopub.status.idle":"2023-04-08T16:33:29.885222Z","shell.execute_reply.started":"2023-04-08T16:33:29.785542Z","shell.execute_reply":"2023-04-08T16:33:29.884249Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_name_or_path = \"microsoft/deberta-v3-base\"\nmax_length = 512\n\ntokenizer = AutoTokenizer.from_pretrained(model_name_or_path)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T16:33:29.925866Z","iopub.execute_input":"2023-04-08T16:33:29.926662Z","iopub.status.idle":"2023-04-08T16:33:31.278489Z","shell.execute_reply.started":"2023-04-08T16:33:29.926617Z","shell.execute_reply":"2023-04-08T16:33:31.277255Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset = PretrainingDataset(\n    texts=texts, \n    tokenizer=tokenizer, \n    max_length=max_length,\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T16:33:31.280773Z","iopub.execute_input":"2023-04-08T16:33:31.281152Z","iopub.status.idle":"2023-04-08T16:33:31.375420Z","shell.execute_reply.started":"2023-04-08T16:33:31.281112Z","shell.execute_reply":"2023-04-08T16:33:31.374223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import sys\nsys.path.append(\"/kaggle/input/pretraining/pretraining-main/src\")","metadata":{"execution":{"iopub.status.busy":"2023-04-08T16:34:33.615756Z","iopub.execute_input":"2023-04-08T16:34:33.616180Z","iopub.status.idle":"2023-04-08T16:34:33.621547Z","shell.execute_reply.started":"2023-04-08T16:34:33.616128Z","shell.execute_reply":"2023-04-08T16:34:33.620294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers import AutoModelForMaskedLM, TrainingArguments, Trainer, DataCollatorForLanguageModeling\n\nmodel = AutoModelForMaskedLM.from_pretrained(model_name_or_path)\n\ndata_collator = DataCollatorForLanguageModeling(\n    tokenizer = tokenizer,\n    mlm = True,\n    mlm_probability = 0.15,\n)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T16:36:00.316116Z","iopub.execute_input":"2023-04-08T16:36:00.316542Z","iopub.status.idle":"2023-04-08T16:36:16.742429Z","shell.execute_reply.started":"2023-04-08T16:36:00.316500Z","shell.execute_reply":"2023-04-08T16:36:16.741268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nfrom torch import nn\nfrom torch.nn import functional as F\nfrom torch.utils.data import DataLoader\nfrom torch.utils.checkpoint import checkpoint\nfrom transformers import AutoModel, AutoConfig\nfrom torchmetrics import functional as metrics\nfrom pytorch_lightning import LightningModule, Trainer\nfrom pretraining.data_collators import MaskedLanguageModelingDataCollator\nimport math\nimport os","metadata":{"execution":{"iopub.status.busy":"2023-04-08T16:37:06.365813Z","iopub.execute_input":"2023-04-08T16:37:06.366211Z","iopub.status.idle":"2023-04-08T16:37:07.808199Z","shell.execute_reply.started":"2023-04-08T16:37:06.366165Z","shell.execute_reply":"2023-04-08T16:37:07.807235Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MaskedLanguageModelingModel(LightningModule):\n    def __init__(self, model_name_or_path, tokenizer, config=None, ignore_index=-100, gradient_checkpointing=False):\n        super().__init__()\n        \n        self.ignore_index = ignore_index\n        self.config = config\n        self.token_embeddings_size = len(tokenizer)\n        \n        if self.config is None:\n            self.config = AutoConfig.from_pretrained(model_name_or_path)\n        \n        self.config.output_hidden_states = True\n        \n        self.backbone = AutoModel.from_pretrained(model_name_or_path, config=self.config)\n        self.backbone.resize_token_embeddings(self.token_embeddings_size)\n        \n        self.head = nn.Linear(in_features=self.config.hidden_size, out_features=self.token_embeddings_size)\n        \n        if gradient_checkpointing:\n            self.backbone.gradient_checkpointing_enable()\n            print(f\"Gradient Checkpointing: {self.backbone.is_gradient_checkpointing}\")\n        \n        self.save_hyperparameters()\n        \n    def forward(self, input_ids, attention_mask=None, **kwargs):\n        backbone_outputs = self.backbone(\n            input_ids=input_ids, \n            attention_mask=attention_mask, \n            **kwargs,\n        )\n        \n        hidden_states = backbone_outputs.hidden_states\n        hidden_state = hidden_states[-1]\n        features = hidden_state[:,0,:]\n        outputs = self.head(features)\n        \n        return outputs\n        \n    def training_step(self, batch, batch_index):\n        input_ids = batch[\"input_ids\"].to(torch.int32)\n        attention_mask = batch[\"attention_mask\"].to(torch.int32)\n        labels = batch[\"labels\"].to(torch.float16)\n        \n        outputs = self(input_ids=input_ids, attention_mask=attention_mask)\n        \n        loss = F.cross_entropy(input=outputs, target=labels, ignore_index=self.ignore_index)\n        perplexity = math.exp(loss)\n        \n        # accuracy\n        predictions = torch.softmax(outputs, dim=-1)\n        accuracy = self.compute_accuracy(predictions, labels)\n        \n        logs = {\n            \"train/loss\": loss,\n            \"train/perplexity\": perplexity,\n            \"train/accuracy\": accuracy,\n        }\n        \n        self.log_dict(logs, prog_bar=False, on_step=True, on_epoch=True)\n        \n        return loss\n    \n    def validation_step(self, batch, batch_index):\n        input_ids = batch[\"input_ids\"].to(torch.int32)\n        attention_mask = batch[\"attention_mask\"].to(torch.int32)\n        labels = batch[\"labels\"].to(torch.float16)\n        \n        outputs = self(input_ids=input_ids, attention_mask=attention_mask)\n        \n        return {\n            \"outputs\": outputs,\n            \"labels\": labels,\n        }\n    \n    def validation_epoch_end(self, validation_outputs):\n        outputs = torch.cat([output[\"outputs\"] for output in validation_outputs], dim=0)\n        labels = torch.cat([output[\"labels\"] for output in validation_outputs], dim=0)\n        \n        loss = F.cross_entropy(input=outputs, target=labels, ignore_index=self.ignore_index)\n        perplexity = math.exp(loss)\n        \n        # accuracy\n        predictions = torch.softmax(outputs, dim=-1)\n        accuracy = self.compute_accuracy(predictions, labels)\n        \n        logs = {\n            \"validation/loss\": loss,\n            \"validation/perplexity\": perplexity,\n            \"validation/accuracy\": accuracy,\n        }\n\n        self.log_dict(logs, prog_bar=False, on_step=False, on_epoch=True)\n        \n    def predict_step(self, batch, batch_index):\n        input_ids = batch[\"input_ids\"].to(torch.int32)\n        attention_mask = batch[\"attention_mask\"].to(torch.int32)\n        \n        outputs = self(input_ids=input_ids, attention_mask=attention_mask)\n        \n        return outputs\n    \n    def compute_accuracy(self, predictions, labels):\n        predictions = predictions.view(-1)\n        labels = labels.view(-1)\n        mask = labels != self.ignore_index\n        predictions, labels = predictions[mask], labels[mask]\n        \n        accuracy = metrics.accuracy(predictions, labels)\n        \n        return accuracy","metadata":{"execution":{"iopub.status.busy":"2023-04-08T16:48:44.647749Z","iopub.execute_input":"2023-04-08T16:48:44.649040Z","iopub.status.idle":"2023-04-08T16:48:44.673284Z","shell.execute_reply.started":"2023-04-08T16:48:44.648976Z","shell.execute_reply":"2023-04-08T16:48:44.671234Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_collator = MaskedLanguageModelingDataCollator(\n    input_key=\"input_ids\", \n    label_key=\"label\",\n    tokenizer=tokenizer,\n    special_tokens_mask_key=\"special_tokens_mask\", \n    masking_probability=0.15,\n    padding_keys=[\"input_ids\", \"attention_mask\", \"special_tokens_mask\"],\n    padding_values=[tokenizer.pad_token_id, 1, 1],\n)\n\ndataloader = DataLoader(\n    dataset=dataset, \n    collate_fn=data_collator,\n)\n\nmodel = MaskedLanguageModelingModel(\n    model_name_or_path=model_name_or_path,\n    tokenizer=tokenizer, \n    gradient_checkpointing=False,\n)\n\n# trainer = Trainer(...)\n# trainer.fit(model=model, train_dataloaders=[dataloader], ckpt_path=None)","metadata":{"execution":{"iopub.status.busy":"2023-04-08T16:57:21.300124Z","iopub.execute_input":"2023-04-08T16:57:21.300569Z","iopub.status.idle":"2023-04-08T16:57:27.383477Z","shell.execute_reply.started":"2023-04-08T16:57:21.300536Z","shell.execute_reply":"2023-04-08T16:57:27.382410Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}