{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"#!pip install transformers","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:37.475765Z","iopub.execute_input":"2022-08-05T00:40:37.476462Z","iopub.status.idle":"2022-08-05T00:40:37.484389Z","shell.execute_reply.started":"2022-08-05T00:40:37.476404Z","shell.execute_reply":"2022-08-05T00:40:37.483495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Import libraries","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport torch\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import DebertaTokenizer, DebertaForSequenceClassification\nfrom sklearn.model_selection import train_test_split\nimport pytorch_lightning as pl\nfrom torch.optim import Adam\nfrom torch.optim.lr_scheduler import StepLR","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:37.493291Z","iopub.execute_input":"2022-08-05T00:40:37.494295Z","iopub.status.idle":"2022-08-05T00:40:39.724451Z","shell.execute_reply.started":"2022-08-05T00:40:37.494256Z","shell.execute_reply":"2022-08-05T00:40:39.723018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load Data","metadata":{}},{"cell_type":"code","source":"train_data_path = \"../input/feedback-prize-effectiveness/train.csv\"\ntest_data_path = \"../input/feedback-prize-effectiveness/test.csv\"\ntrain_df = pd.read_csv(train_data_path)\ntest_df = pd.read_csv(test_data_path)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:39.726025Z","iopub.execute_input":"2022-08-05T00:40:39.726899Z","iopub.status.idle":"2022-08-05T00:40:39.944671Z","shell.execute_reply.started":"2022-08-05T00:40:39.726853Z","shell.execute_reply":"2022-08-05T00:40:39.943520Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[:2]","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:39.948178Z","iopub.execute_input":"2022-08-05T00:40:39.948601Z","iopub.status.idle":"2022-08-05T00:40:39.974460Z","shell.execute_reply.started":"2022-08-05T00:40:39.948561Z","shell.execute_reply":"2022-08-05T00:40:39.972103Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## config","metadata":{}},{"cell_type":"code","source":"# ------ Model ------\nMODEL_NAME = \"microsoft/deberta-base\"\nTOKENIZER_PATH = \"microsoft/deberta-base\"\n# ------ Training ------\nBATCH_SIZE = 4\nLEARNING_RATE = 1e-3\nSTEP_SIZE = 1","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:39.976131Z","iopub.execute_input":"2022-08-05T00:40:39.977960Z","iopub.status.idle":"2022-08-05T00:40:39.988857Z","shell.execute_reply.started":"2022-08-05T00:40:39.977919Z","shell.execute_reply":"2022-08-05T00:40:39.986885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make dataset","metadata":{}},{"cell_type":"code","source":"train_id, train_text, train_label = train_df[\"discourse_id\"].tolist(), train_df[\"discourse_text\"].tolist(), train_df[\"discourse_effectiveness\"].tolist()\n(train_id, val_id,\ntrain_text, val_text,\ntrain_label, val_label) = train_test_split(train_id, train_text, train_label, test_size=0.2, random_state=37)\ntest_id, test_text = test_df[\"discourse_id\"].tolist(), test_df[\"discourse_text\"].tolist()\nprint(len(train_id), len(val_id), len(test_id))","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:39.992468Z","iopub.execute_input":"2022-08-05T00:40:39.993778Z","iopub.status.idle":"2022-08-05T00:40:40.061479Z","shell.execute_reply.started":"2022-08-05T00:40:39.993736Z","shell.execute_reply":"2022-08-05T00:40:40.060352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DataSetforTrain(Dataset):\n    def __init__(self, ids, texts, labels):\n        self.ids = ids\n        self.texts = texts\n        self.labels = labels\n        self.label_dict = {\"Ineffective\":0, \"Adequate\": 1, \"Effective\": 2}\n        \n    def __len__(self):\n        return len(self.ids)\n    \n    def __getitem__(self, idx):\n        id = self.ids[idx]\n        text = self.texts[idx]\n        label = self.label_dict[self.labels[idx]]\n        return id, text, label\n    \nclass DataSetforTest(Dataset):\n    def __init__(self, ids, texts):\n        self.ids = ids\n        self.texts = texts\n        \n    def __len__(self):\n        return len(self.ids)\n    \n    def __getitem__(self, idx):\n        id = self.ids[idx]\n        text = self.texts[idx]\n        return id, text","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:40.066332Z","iopub.execute_input":"2022-08-05T00:40:40.069053Z","iopub.status.idle":"2022-08-05T00:40:40.086147Z","shell.execute_reply.started":"2022-08-05T00:40:40.069009Z","shell.execute_reply":"2022-08-05T00:40:40.084837Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = DataSetforTrain(train_id, train_text, train_label)\nval_dataset = DataSetforTrain(val_id, val_text, val_label)\ntest_dataset = DataSetforTest(test_id, test_text)\nlen(train_dataset), len(val_dataset), len(test_dataset)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:40.091846Z","iopub.execute_input":"2022-08-05T00:40:40.094376Z","iopub.status.idle":"2022-08-05T00:40:40.107325Z","shell.execute_reply.started":"2022-08-05T00:40:40.094319Z","shell.execute_reply":"2022-08-05T00:40:40.106061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset[0]","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:40.113614Z","iopub.execute_input":"2022-08-05T00:40:40.114185Z","iopub.status.idle":"2022-08-05T00:40:40.126785Z","shell.execute_reply.started":"2022-08-05T00:40:40.114145Z","shell.execute_reply":"2022-08-05T00:40:40.125335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make DataLoader","metadata":{}},{"cell_type":"code","source":"tokenizer = DebertaTokenizer.from_pretrained(TOKENIZER_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:40.128027Z","iopub.execute_input":"2022-08-05T00:40:40.128727Z","iopub.status.idle":"2022-08-05T00:40:43.005894Z","shell.execute_reply.started":"2022-08-05T00:40:40.128659Z","shell.execute_reply":"2022-08-05T00:40:43.004771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_texts = [\"Hello, my dog is cute\", \"Hello world\"]\ninputs = tokenizer(sample_texts, padding=True, return_tensors=\"pt\")\ninputs","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:43.008032Z","iopub.execute_input":"2022-08-05T00:40:43.009621Z","iopub.status.idle":"2022-08-05T00:40:43.020132Z","shell.execute_reply.started":"2022-08-05T00:40:43.009580Z","shell.execute_reply":"2022-08-05T00:40:43.019016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_fn_for_train(batch):\n    ids, texts, labels = list(zip(*batch))\n    texts = tokenizer(texts, padding=True, return_tensors=\"pt\")\n    labels = torch.tensor(labels)\n    return ids, texts, labels\n\ndef collate_fn_for_test(batch):\n    ids, texts = list(zip(*batch))\n    texts = tokenizer(texts, padding=True, return_tensors=\"pt\")\n    return ids, texts","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:43.021979Z","iopub.execute_input":"2022-08-05T00:40:43.023425Z","iopub.status.idle":"2022-08-05T00:40:43.032537Z","shell.execute_reply.started":"2022-08-05T00:40:43.023381Z","shell.execute_reply":"2022-08-05T00:40:43.031503Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataloader = DataLoader(train_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn_for_train)\nval_dataloader = DataLoader(val_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn_for_train)\ntest_dataloader = DataLoader(test_dataset, batch_size=BATCH_SIZE, collate_fn=collate_fn_for_test)\nlen(train_dataloader), len(val_dataloader), len(test_dataloader)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:43.036773Z","iopub.execute_input":"2022-08-05T00:40:43.037674Z","iopub.status.idle":"2022-08-05T00:40:43.048860Z","shell.execute_reply.started":"2022-08-05T00:40:43.037610Z","shell.execute_reply":"2022-08-05T00:40:43.047635Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Make Model","metadata":{}},{"cell_type":"code","source":"class Net(pl.LightningModule):\n    def __init__(self):\n        super().__init__()\n        self.model = DebertaForSequenceClassification.from_pretrained(MODEL_NAME, num_labels=3)\n        \n    def forward(self, x):\n        return self.model(x)\n    \n    def training_step(self, batch, batch_idx):\n        ids, inputs, labels = batch\n        loss = self.model(**inputs, labels=labels).loss\n        return loss\n    \n    def validation_step(self, batch, batch_idx):\n        ids, inputs, labels = batch\n        loss = self.model(**inputs, labels=labels).loss\n        return {\"val_loss\": loss}\n    \n    def validation_epo_end(self, outputs):\n        avg_loss = torch.stack([x[\"val_loss\"] for x in outputs]).mean()\n        return {\"avg_val_loss\": avg_loss}\n    \n    def configure_optimizers(self):\n        optimizer = torch.optim.Adam(self.parameters(), lr=LEARNING_RATE)\n        return [optimizer], StepLR(optimizer, step_size=STEP_SIZE)\n    ","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:43.050409Z","iopub.execute_input":"2022-08-05T00:40:43.050776Z","iopub.status.idle":"2022-08-05T00:40:43.062235Z","shell.execute_reply.started":"2022-08-05T00:40:43.050741Z","shell.execute_reply":"2022-08-05T00:40:43.061079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{}},{"cell_type":"code","source":"net = Net()\ntrainer = pl.Trainer(gpus=1)\ntrainer.fit(net, train_dataloader, val_dataloader)","metadata":{"execution":{"iopub.status.busy":"2022-08-05T00:40:43.063882Z","iopub.execute_input":"2022-08-05T00:40:43.064827Z","iopub.status.idle":"2022-08-05T01:53:22.474908Z","shell.execute_reply.started":"2022-08-05T00:40:43.064783Z","shell.execute_reply":"2022-08-05T01:53:22.473897Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}