{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 📝 Versions\nVersion 5: DeBertaV3 **CV:0.680 LB: 0.**\n\nTraining Notebook for DeBertaV3 base [Training Notebook](https://www.kaggle.com/code/bharadwajvedula/pytorch-gpu-debertav3-base-training-fold-4/)","metadata":{}},{"cell_type":"markdown","source":"# 🚚 Imports","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nfrom tqdm.auto import tqdm\n\nimport numpy as np \nimport pandas as pd \n\nfrom text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\nfrom sklearn.metrics import log_loss\n\nfrom transformers import AutoModel, AutoTokenizer, AdamW, DataCollatorWithPadding\n\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning import Trainer, seed_everything\nfrom pytorch_lightning.callbacks import ModelCheckpoint, EarlyStopping","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-28T13:59:21.018829Z","iopub.execute_input":"2022-07-28T13:59:21.019995Z","iopub.status.idle":"2022-07-28T13:59:30.324579Z","shell.execute_reply.started":"2022-07-28T13:59:21.019877Z","shell.execute_reply":"2022-07-28T13:59:30.323500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# ⚙️ Config","metadata":{}},{"cell_type":"code","source":"class config:\n    base_dir = \"../input/feedback-prize-effectiveness/\"\n    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n    seed = 42\n    # dataset path \n    train_dataset_path = \"../input/feedbackprizegroupkfolds/train.csv\"\n    test_dataset_path = \"../input/feedback-prize-effectiveness/test.csv\"\n    sample_submission_path = \"../input/feedback-prize-effectiveness/sample_submission.csv\"\n       \n    save_dir=\"./result\"\n    \n    #tokenizer params\n    truncation = True \n    padding = 'max_length'\n    max_length = 512\n    tokenizer_path = \"../input/feedbackprizedebertav3fold4/result/tokenizer\"\n    \n    # model params\n    model_name = \"microsoft/deberta-v3-base\"\n    hf_model = \"../input/feedbackprizedebertav3fold4/result/hf_model\"\n    \n    #training params\n    learning_rate = 1e-5\n    batch_size = 4\n    epochs = 12\n\nseed_everything(config.seed)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:59:30.326888Z","iopub.execute_input":"2022-07-28T13:59:30.327880Z","iopub.status.idle":"2022-07-28T13:59:30.407537Z","shell.execute_reply.started":"2022-07-28T13:59:30.327840Z","shell.execute_reply":"2022-07-28T13:59:30.406632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📊 Preprocessing","metadata":{}},{"cell_type":"code","source":"def get_test_essay(essay_id):\n    parent_path = config.base_dir + 'test'\n    essay_path = os.path.join(parent_path, f\"{essay_id}.txt\")\n    essay_text = open(essay_path, 'r').read()\n    return essay_text","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:59:30.409306Z","iopub.execute_input":"2022-07-28T13:59:30.409994Z","iopub.status.idle":"2022-07-28T13:59:30.416857Z","shell.execute_reply.started":"2022-07-28T13:59:30.409957Z","shell.execute_reply":"2022-07-28T13:59:30.415870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n# Register the encoding and decoding error handlers for `utf-8` and `cp1252`.\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n    \"\"\"Resolve the encoding problems and normalize the abnormal characters.\"\"\"\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    text = unidecode(text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:59:30.420541Z","iopub.execute_input":"2022-07-28T13:59:30.420884Z","iopub.status.idle":"2022-07-28T13:59:30.430069Z","shell.execute_reply.started":"2022-07-28T13:59:30.420858Z","shell.execute_reply":"2022-07-28T13:59:30.428861Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train = pd.read_csv(config.train_dataset_path)\n\ndf_test = pd.read_csv(config.test_dataset_path)\ndf_ss = pd.read_csv(config.sample_submission_path)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:59:30.431585Z","iopub.execute_input":"2022-07-28T13:59:30.432226Z","iopub.status.idle":"2022-07-28T13:59:30.710045Z","shell.execute_reply.started":"2022-07-28T13:59:30.432188Z","shell.execute_reply":"2022-07-28T13:59:30.709061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train['essay_text'] = df_train['essay_id'].apply(get_train_essay)\n\ndf_test['essay_text'] = df_test['essay_id'].apply(get_test_essay)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T13:59:30.711510Z","iopub.execute_input":"2022-07-28T13:59:30.711867Z","iopub.status.idle":"2022-07-28T14:00:02.684222Z","shell.execute_reply.started":"2022-07-28T13:59:30.711823Z","shell.execute_reply":"2022-07-28T14:00:02.683263Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train['discourse_text'] = df_train['discourse_text'].apply(resolve_encodings_and_normalize)\n# df_train['essay_text'] = df_train['essay_text'].apply(resolve_encodings_and_normalize)\n\n\ndf_test['discourse_text'] = df_test['discourse_text'].apply(resolve_encodings_and_normalize)\ndf_test['essay_text'] = df_test['essay_text'].apply(resolve_encodings_and_normalize)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:00:02.687417Z","iopub.execute_input":"2022-07-28T14:00:02.687720Z","iopub.status.idle":"2022-07-28T14:00:26.730928Z","shell.execute_reply.started":"2022-07-28T14:00:02.687695Z","shell.execute_reply":"2022-07-28T14:00:26.729919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# df_train['text'] = df_train['discourse_type'] + \" [SEP] \" + df_train['discourse_text'] + \" [SEP] \" + df_train['essay_text']\n\ndf_test['text'] = df_test['discourse_type'] + \" [SEP] \" + df_test['discourse_text'] + \" [SEP] \" + df_test['essay_text']","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:00:26.732675Z","iopub.execute_input":"2022-07-28T14:00:26.733241Z","iopub.status.idle":"2022-07-28T14:00:26.807185Z","shell.execute_reply.started":"2022-07-28T14:00:26.733203Z","shell.execute_reply":"2022-07-28T14:00:26.806215Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🎟 Tokenizer","metadata":{}},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_path)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:00:26.808684Z","iopub.execute_input":"2022-07-28T14:00:26.809088Z","iopub.status.idle":"2022-07-28T14:00:27.155109Z","shell.execute_reply.started":"2022-07-28T14:00:26.809050Z","shell.execute_reply":"2022-07-28T14:00:27.154127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🧰 Dataset Prep Function","metadata":{}},{"cell_type":"code","source":"class FeedbackPrizeDataset(Dataset):\n    def __init__(self, text):\n        self.tokenizer = AutoTokenizer.from_pretrained(config.tokenizer_path)\n        self.text = text\n    \n    def __len__(self):\n        return len(self.text)\n    \n    def __getitem__(self, idx):\n        text = self.text[idx]\n        test_embeddings = tokenizer.encode_plus(\n            text,\n            truncation = config.truncation,\n            max_length = config.max_length, \n        )\n    \n        return {'input_ids':test_embeddings['input_ids'],\n                'attention_mask': test_embeddings['attention_mask'],\n                }","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:00:27.158787Z","iopub.execute_input":"2022-07-28T14:00:27.159127Z","iopub.status.idle":"2022-07-28T14:00:27.165656Z","shell.execute_reply.started":"2022-07-28T14:00:27.159101Z","shell.execute_reply":"2022-07-28T14:00:27.164743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Collate:\n    def __init__(self, tokenizer, isTrain=True):\n        self.tokenizer = tokenizer\n        self.isTrain = isTrain\n\n    def __call__(self, batch):\n        output = dict()\n        output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n        output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n        if self.isTrain:\n            output[\"target\"] = [sample[\"target\"] for sample in batch]\n\n        # calculate max token length of this batch\n        batch_max = max([len(ids) for ids in output[\"input_ids\"]])\n\n        # add padding\n        if self.tokenizer.padding_side == \"right\":\n            output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n        else:\n            output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n\n        # convert to tensors\n        output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n        output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n        if self.isTrain:\n            output[\"target\"] = torch.tensor(output[\"target\"], dtype=torch.long)\n\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:00:27.167077Z","iopub.execute_input":"2022-07-28T14:00:27.168106Z","iopub.status.idle":"2022-07-28T14:00:27.181824Z","shell.execute_reply.started":"2022-07-28T14:00:27.168070Z","shell.execute_reply":"2022-07-28T14:00:27.180976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🧠 Model","metadata":{}},{"cell_type":"code","source":"class FeedbackPrizeModel(pl.LightningModule):\n    def __init__(self, train_dataloader, validation_dataloader):\n        super().__init__()\n        self.transformers_model = AutoModel.from_pretrained(config.hf_model)\n        self.classifier =  nn.Linear(self.transformers_model.config.hidden_size,3)\n        self.loss_function = nn.CrossEntropyLoss()\n        self._train_dataloader = train_dataloader\n        self._validation_dataloader = validation_dataloader\n        self.save_hyperparameters()\n        \n    def forward(self, input_ids, attention_mask):\n        cls_token = self.transformers_model(input_ids, attention_mask)[0][:,0,:]\n        output = self.classifier(cls_token)\n        return output\n    \n    def training_step(self,batch,batch_idx):\n        input_ids = batch['input_ids']\n        attention_mask = batch['attention_mask']\n        target = batch['target']\n        output = self(input_ids,attention_mask)\n        loss = self.loss_function(output, target)\n        self.log('train_loss', loss , prog_bar=True)\n        return {'loss': loss}\n    \n    def train_epoch_end(self,outputs):\n        avg_loss = torch.stack([x['loss'] for x in outputs]).mean()\n        print(f'epoch {trainer.current_epoch} training loss {avg_loss}')\n        return {'train_loss': avg_loss} \n    \n    def validation_step(self,batch,batch_idx):\n        input_ids = batch['input_ids']\n        attention_mask = batch['attention_mask']\n        target = batch['target']\n        output = self(input_ids,attention_mask)\n        loss = self.loss_function(output, target)\n        self.log('val_loss', loss , prog_bar=True)\n        return {'val_loss': loss}        \n\n    \n    def validation_epoch_end(self,outputs):\n        avg_loss = torch.stack([x['val_loss'] for x in outputs]).mean()\n        print(f'epoch {trainer.current_epoch} validation loss {avg_loss}')\n        return {'val_loss': avg_loss}\n    \n    def test_step(self):\n        input_ids = batch['input_ids']\n        attention_mask = batch['attention_mask']\n        target = batch['target']\n        output = self(input_ids , attention_mask)\n        loss = self.loss_function(output, target)\n        self.log('test_loss', loss)\n        return {'test_loss': loss}\n    \n    def test_epoch_end(self):\n        avg_loss = torch.stack([x['test_loss'] for x in outputs]).mean()\n        print(f'epoch {trainer.current_epoch} test loss {avg_loss}')\n        return {'test_loss': avg_loss}\n        \n    def train_dataloader(self):\n        return self._train_dataloader \n    \n    def validation_dataloader(self):\n        return self._validation_dataloader\n    \n    def configure_optimizers(self):\n        optimizer = AdamW(self.parameters(), lr = config.learning_rate)\n        return [optimizer]\n  ","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:00:27.183289Z","iopub.execute_input":"2022-07-28T14:00:27.183629Z","iopub.status.idle":"2022-07-28T14:00:27.203244Z","shell.execute_reply.started":"2022-07-28T14:00:27.183597Z","shell.execute_reply":"2022-07-28T14:00:27.202354Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef predict(data_loader, model):\n        \n    model.to(config.device)\n    model.eval()\n    model.zero_grad()\n    \n    predictions = []\n    for batch in tqdm(data_loader):\n        ids = batch['input_ids'].to(config.device, dtype = torch.long)\n        mask = batch['attention_mask'].to(config.device, dtype = torch.long)\n        with torch.no_grad():\n            outputs = model(input_ids = ids, attention_mask = mask)\n        outputs = F.softmax(outputs, dim=1)\n        predictions.extend(outputs.detach().cpu().numpy())\n        \n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:00:27.204722Z","iopub.execute_input":"2022-07-28T14:00:27.205341Z","iopub.status.idle":"2022-07-28T14:00:27.215621Z","shell.execute_reply.started":"2022-07-28T14:00:27.205305Z","shell.execute_reply":"2022-07-28T14:00:27.214731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 🔄 KFold Predictions","metadata":{}},{"cell_type":"code","source":"predictions = []\nfor fold in range(1,5):\n    print(f\"====== FOLD RUNNING {fold}======\")\n\n    X_test = df_test['text'].tolist()\n#     X_test = df_train.loc[df_train['kfold'] != fold]['text'].tolist()\n    \n    print(\"Generating Test Dataset\")\n    test_dataset = FeedbackPrizeDataset(X_test)\n\n    print(\"Generating Test DataLoader\")\n    test_dataloader = DataLoader(test_dataset, batch_size = config.batch_size, shuffle = False, num_workers= 2, pin_memory=True,collate_fn = Collate(tokenizer, isTrain=False))\n\n    print(\"prediction on Test data\")\n    path = f'../input/feedbackprizedebertav3fold{fold}/result/microsoft/deberta-v3-base_{fold}.ckpt'\n    model = FeedbackPrizeModel.load_from_checkpoint(path)\n    preds = predict(test_dataloader, model)    \n    predictions.append(preds)\n    \n    del model,test_dataloader\n    torch.cuda.empty_cache()\n    gc.collect()\n    ","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:00:27.217125Z","iopub.execute_input":"2022-07-28T14:00:27.217600Z","iopub.status.idle":"2022-07-28T14:28:35.705959Z","shell.execute_reply.started":"2022-07-28T14:00:27.217558Z","shell.execute_reply":"2022-07-28T14:28:35.701720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Post Processing","metadata":{}},{"cell_type":"code","source":"final_predictions = np.mean([predictions[0],predictions[1],predictions[2],predictions[3]] , axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:28:35.707578Z","iopub.status.idle":"2022-07-28T14:28:35.708191Z","shell.execute_reply.started":"2022-07-28T14:28:35.707922Z","shell.execute_reply":"2022-07-28T14:28:35.707947Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"adequate = []\neffective = []\nineffective = []\n\nfor prediction in final_predictions:\n    adequate.append(prediction[0])\n    effective.append(prediction[1])\n    ineffective.append(prediction[2])","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:28:35.711678Z","iopub.status.idle":"2022-07-28T14:28:35.712264Z","shell.execute_reply.started":"2022-07-28T14:28:35.712013Z","shell.execute_reply":"2022-07-28T14:28:35.712037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 💯Submission","metadata":{}},{"cell_type":"code","source":"submission = pd.DataFrame({'discourse_id':df_test['discourse_id'],'Adequate':adequate,'Effective':effective,'Ineffective':ineffective})\nsubmission.to_csv(\"submission.csv\",index = False)","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:28:35.714352Z","iopub.status.idle":"2022-07-28T14:28:35.715723Z","shell.execute_reply.started":"2022-07-28T14:28:35.715317Z","shell.execute_reply":"2022-07-28T14:28:35.715344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-28T14:28:35.717370Z","iopub.status.idle":"2022-07-28T14:28:35.717993Z","shell.execute_reply.started":"2022-07-28T14:28:35.717669Z","shell.execute_reply":"2022-07-28T14:28:35.717694Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 📍 References\n\n1. [Notebook 1](https://www.kaggle.com/code/pypiahmad/deberta-base-large-roberta/notebook)","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}