{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Import","metadata":{"id":"KaPlO6DhaXHq"}},{"cell_type":"code","source":"!pip install wandb\n!wandb login 8ce328c0bf5e805a7469a901916c9dcd44c83c14","metadata":{"execution":{"iopub.status.busy":"2022-07-30T07:53:27.630875Z","iopub.execute_input":"2022-07-30T07:53:27.631285Z","iopub.status.idle":"2022-07-30T07:53:38.842082Z","shell.execute_reply.started":"2022-07-30T07:53:27.631249Z","shell.execute_reply":"2022-07-30T07:53:38.841172Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.nn as nn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import log_loss\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, LearningRateMonitor\nfrom transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoConfig, get_cosine_schedule_with_warmup, get_linear_schedule_with_warmup\n\nfrom pytorch_lightning.loggers import WandbLogger\nfrom pytorch_lightning import Trainer\n\nwandb_logger = WandbLogger(project=\"COMP9417\")","metadata":{"id":"5MrIl3-eaXHq","execution":{"iopub.status.busy":"2022-07-30T07:53:38.845673Z","iopub.execute_input":"2022-07-30T07:53:38.846561Z","iopub.status.idle":"2022-07-30T07:53:38.857284Z","shell.execute_reply.started":"2022-07-30T07:53:38.846520Z","shell.execute_reply":"2022-07-30T07:53:38.856261Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{"id":"hnptXakgaXHs"}},{"cell_type":"code","source":"\ntrain = pd.read_csv('../input/feedback-prize-effectiveness/train.csv')\ntest = pd.read_csv('../input/feedback-prize-effectiveness/test.csv')\ntrain.shape, test.shape","metadata":{"id":"JFDPaO9DaXHs","outputId":"86404ded-93eb-4b3a-bbbc-27caee60e560","execution":{"iopub.status.busy":"2022-07-30T07:53:38.858991Z","iopub.execute_input":"2022-07-30T07:53:38.859653Z","iopub.status.idle":"2022-07-30T07:53:39.121111Z","shell.execute_reply.started":"2022-07-30T07:53:38.859618Z","shell.execute_reply":"2022-07-30T07:53:39.120389Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(5)","metadata":{"id":"YyVD5EokaXHt","outputId":"8daa2af8-b5b4-4887-c411-1d478cf9698d","execution":{"iopub.status.busy":"2022-07-30T07:53:39.125792Z","iopub.execute_input":"2022-07-30T07:53:39.127871Z","iopub.status.idle":"2022-07-30T07:53:39.142814Z","shell.execute_reply.started":"2022-07-30T07:53:39.127831Z","shell.execute_reply":"2022-07-30T07:53:39.142057Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"id":"hMPtnjpvaXHt","outputId":"9184b15a-a0f7-48fa-ae57-ef6e08726014","execution":{"iopub.status.busy":"2022-07-30T07:53:39.144182Z","iopub.execute_input":"2022-07-30T07:53:39.144660Z","iopub.status.idle":"2022-07-30T07:53:39.160463Z","shell.execute_reply.started":"2022-07-30T07:53:39.144621Z","shell.execute_reply":"2022-07-30T07:53:39.159085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_values = train['discourse_type'].value_counts()\ntype_values","metadata":{"id":"O1vDF80MaXHu","outputId":"6f83c26c-a013-48e7-e592-f04eb749310d","execution":{"iopub.status.busy":"2022-07-30T07:53:39.162074Z","iopub.execute_input":"2022-07-30T07:53:39.162492Z","iopub.status.idle":"2022-07-30T07:53:39.180107Z","shell.execute_reply.started":"2022-07-30T07:53:39.162459Z","shell.execute_reply":"2022-07-30T07:53:39.179463Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_values = train['discourse_effectiveness'].value_counts()\ntarget_values","metadata":{"id":"e0scIeoJaXHu","outputId":"273ece24-22fe-4dc8-82ec-4d232fecbf10","execution":{"iopub.status.busy":"2022-07-30T07:53:39.184819Z","iopub.execute_input":"2022-07-30T07:53:39.186949Z","iopub.status.idle":"2022-07-30T07:53:39.202902Z","shell.execute_reply.started":"2022-07-30T07:53:39.186914Z","shell.execute_reply":"2022-07-30T07:53:39.202179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess","metadata":{"id":"FdTqUoT0aXHv"}},{"cell_type":"code","source":"train_dir = \"../input/feedback-prize-effectiveness/train\"\ntest_dir = \"../input/feedback-prize-effectiveness/test\"\n\ndef get_essay(eassy_id, dir):\n    essay_dir = os.path.join(dir, f'{eassy_id}.txt')\n    essay_txt = open(essay_dir, 'r').read()\n    return essay_txt\n","metadata":{"id":"I3-e3LQTaXHw","execution":{"iopub.status.busy":"2022-07-30T07:53:39.204211Z","iopub.execute_input":"2022-07-30T07:53:39.204762Z","iopub.status.idle":"2022-07-30T07:53:39.211474Z","shell.execute_reply.started":"2022-07-30T07:53:39.204727Z","shell.execute_reply":"2022-07-30T07:53:39.210598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['essay_text'] = train['essay_id'].apply(lambda x: get_essay(x, train_dir))\ntest['essay_text'] = test['essay_id'].apply(lambda x: get_essay(x, test_dir))\ntest.head()","metadata":{"id":"VX0-d6epaXHw","outputId":"65613dc8-68e6-4abf-85cb-c5fc799c7b36","execution":{"iopub.status.busy":"2022-07-30T07:53:39.213057Z","iopub.execute_input":"2022-07-30T07:53:39.213825Z","iopub.status.idle":"2022-07-30T07:53:54.238402Z","shell.execute_reply.started":"2022-07-30T07:53:39.213786Z","shell.execute_reply":"2022-07-30T07:53:54.237199Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ntrain['discourse_effectiveness'] = encoder.fit_transform(train['discourse_effectiveness'])\ntrain['discourse_type'] = encoder.fit_transform(train['discourse_type'])\ntest['discourse_type'] = encoder.fit_transform(test['discourse_type'])","metadata":{"id":"SHP0acMeaXHw","execution":{"iopub.status.busy":"2022-07-30T07:53:54.241420Z","iopub.execute_input":"2022-07-30T07:53:54.241696Z","iopub.status.idle":"2022-07-30T07:53:54.272068Z","shell.execute_reply.started":"2022-07-30T07:53:54.241671Z","shell.execute_reply":"2022-07-30T07:53:54.271384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntest.head()","metadata":{"id":"7H4paH-SaXHx","outputId":"1023331f-7250-4720-95e7-17fb53f58078","execution":{"iopub.status.busy":"2022-07-30T07:53:54.273399Z","iopub.execute_input":"2022-07-30T07:53:54.273731Z","iopub.status.idle":"2022-07-30T07:53:54.286701Z","shell.execute_reply.started":"2022-07-30T07:53:54.273698Z","shell.execute_reply":"2022-07-30T07:53:54.286007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{"id":"2e-4-80CaXHx"}},{"cell_type":"code","source":"class EssayDataset(Dataset):\n    def __init__(self, df, tokenizer, max_len=512):\n        self.df = df\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n        self.discourse_type = df['discourse_type'].values\n        self.discourse = df['discourse_text'].values\n        self.essay = df['essay_text'].values\n        if 'discourse_effectiveness' in self.df:\n            self.target = df['discourse_effectiveness'].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        discourse_type = self.discourse_type[idx]\n        discourse = self.discourse[idx]\n        essay = self.essay[idx]\n        text = discourse + \" \" + self.tokenizer.sep_token + \" \" + essay\n\n        encode_dict = self.tokenizer.encode_plus(\n            text,\n            add_special_tokens=True,\n            truncation=True,\n            max_length=self.max_len,\n            padding='max_length',\n            return_attention_mask=True,\n            return_tensors='pt'\n        )\n        ids = encode_dict['input_ids']\n        mask = encode_dict['attention_mask']\n\n        ids = ids.squeeze(0)\n        mask = mask.squeeze(0)\n\n        if 'discourse_effectiveness' in self.df:\n            target = self.target[idx]\n            return {\"ids\" : ids, \"mask\": mask, \"target\": target, \"dense_feature\": discourse_type}\n        return {\"ids\": ids, \"mask\": mask, \"dense_feature\": discourse_type}","metadata":{"id":"v-xj6x7waXHx","execution":{"iopub.status.busy":"2022-07-30T07:53:54.287836Z","iopub.execute_input":"2022-07-30T07:53:54.288876Z","iopub.status.idle":"2022-07-30T07:53:54.300927Z","shell.execute_reply.started":"2022-07-30T07:53:54.288758Z","shell.execute_reply":"2022-07-30T07:53:54.300139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel_name = 'bert-base-uncased'","metadata":{"id":"LeKYJhi9aXHy","execution":{"iopub.status.busy":"2022-07-30T07:53:54.302039Z","iopub.execute_input":"2022-07-30T07:53:54.302381Z","iopub.status.idle":"2022-07-30T07:53:54.319774Z","shell.execute_reply.started":"2022-07-30T07:53:54.302331Z","shell.execute_reply":"2022-07-30T07:53:54.318942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(model_name)\ntokenizer","metadata":{"id":"zRTv9kc-aXHy","outputId":"dd787bbd-b3f6-4db3-ba38-dd310fcfb56e","execution":{"iopub.status.busy":"2022-07-30T07:53:54.320836Z","iopub.execute_input":"2022-07-30T07:53:54.322014Z","iopub.status.idle":"2022-07-30T07:53:57.399130Z","shell.execute_reply.started":"2022-07-30T07:53:54.321978Z","shell.execute_reply":"2022-07-30T07:53:57.398387Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len = 512\ntrain_, valid_ = train_test_split(train, test_size=0.2, random_state=42)\ntrain_dataset = EssayDataset(train_, tokenizer, max_len)\n\nvalid_dataset = EssayDataset(valid_, tokenizer, max_len)\ntest_dataset = EssayDataset(test, tokenizer, max_len)","metadata":{"id":"ZyV0Gh4kaXHz","execution":{"iopub.status.busy":"2022-07-30T07:53:57.400413Z","iopub.execute_input":"2022-07-30T07:53:57.400742Z","iopub.status.idle":"2022-07-30T07:53:57.418751Z","shell.execute_reply.started":"2022-07-30T07:53:57.400714Z","shell.execute_reply":"2022-07-30T07:53:57.418047Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{"id":"iOA1SFZ_aXHz"}},{"cell_type":"code","source":"class FeedBackModel(nn.Module):\n    def __init__(self, model_name):\n        super(FeedBackModel, self).__init__()\n        self.deberta = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=10, output_attentions=False, output_hidden_states=False)\n        self.dropout = nn.Dropout(0.5)\n        self.fc = nn.Linear(10+1, 3)\n        \n    def forward(self, batch):        \n        input_ids, attention_masks, dense_feature = batch['ids'], batch['mask'], batch['dense_feature']\n        out = self.deberta(input_ids, attention_mask=attention_masks, output_hidden_states=False)\n        output = self.dropout(out.logits)\n        dense_feature = dense_feature.reshape((-1, 1))\n        output = torch.concat([out.logits, dense_feature], dim=-1)\n        output = self.fc(output)\n        return output","metadata":{"id":"s_J0KmkZZsci","execution":{"iopub.status.busy":"2022-07-30T07:53:57.420070Z","iopub.execute_input":"2022-07-30T07:53:57.420653Z","iopub.status.idle":"2022-07-30T07:53:57.429795Z","shell.execute_reply.started":"2022-07-30T07:53:57.420617Z","shell.execute_reply":"2022-07-30T07:53:57.428987Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Classifier(pl.LightningModule):\n    def __init__(self, hparams, model):\n        super(Classifier, self).__init__()\n        self.save_hyperparameters(ignore=['model'])\n\n        self.model = model\n        self.batch_size = hparams[\"batch_size\"]\n        self.lr = hparams[\"lr\"]\n        self.wd = hparams['weight_decay']\n        self.steps = hparams['total_steps']\n\n\n    def forward(self, batch):\n        output = self.model(batch)\n        return output\n\n    def configure_optimizers(self):\n        optimizer = optim.AdamW(self.model.parameters(), lr=self.lr, weight_decay=self.wd)\n        scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=int(self.steps * 0.1), num_training_steps=self.steps)\n        scheduler = {\"scheduler\": scheduler, \"interval\": \"step\", \"frequency\": 1}\n        return [optimizer], [scheduler]\n\n    def training_step(self, batch, batch_idx):\n        output = self.forward(batch)\n        pred_flat = torch.argmax(output, dim=1).flatten()\n        labels_flat = batch['target'].flatten()\n        loss = F.cross_entropy(output, labels_flat)\n        acc = torch.sum(pred_flat == labels_flat) / len(labels_flat)\n        self.log(\"train_loss\", loss, on_epoch=False, on_step=True)\n        self.log(\"train_acc\", acc, on_epoch=False, on_step=True)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        output = self.forward(batch)\n        pred_flat = torch.argmax(output, dim=1).flatten()\n        labels_flat = batch['target'].flatten()\n        loss = F.cross_entropy(output, labels_flat)\n        acc = torch.sum(pred_flat == labels_flat) / len(labels_flat)\n        self.log(\"val_loss\", loss)\n        self.log(\"val_acc\", acc)\n\n\n    def train_dataloader(self):\n        train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=2, pin_memory=True)\n        return train_loader\n\n    def val_dataloader(self):\n        valid_loader = DataLoader(valid_dataset, batch_size=self.batch_size, shuffle=False, num_workers=2, pin_memory=True)\n        return valid_loader\n\n    def test_dataloader(self):\n        test_loader = DataLoader(test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=2, pin_memory=True)\n        return test_loader","metadata":{"id":"U4K6-Y661BfG","execution":{"iopub.status.busy":"2022-07-30T07:53:57.432250Z","iopub.execute_input":"2022-07-30T07:53:57.432567Z","iopub.status.idle":"2022-07-30T07:53:57.450397Z","shell.execute_reply.started":"2022-07-30T07:53:57.432543Z","shell.execute_reply":"2022-07-30T07:53:57.449665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{"id":"WyqX08iwaXH0"}},{"cell_type":"code","source":"pl.seed_everything(42)\nhparams = {\n    \"batch_size\": 16,\n    \"lr\": 2e-5,\n    \"weight_decay\": 1e-2,\n    \"epochs\": 3,\n}\n\nloader = DataLoader(train_dataset, batch_size=hparams[\"batch_size\"])\nhparams[\"total_steps\"] = len(loader) * hparams[\"epochs\"]","metadata":{"id":"2rceJup7aXH0","outputId":"c007fb00-3d7d-4274-c15f-843bc82cbb3d","execution":{"iopub.status.busy":"2022-07-30T07:53:57.453388Z","iopub.execute_input":"2022-07-30T07:53:57.453642Z","iopub.status.idle":"2022-07-30T07:53:57.463373Z","shell.execute_reply.started":"2022-07-30T07:53:57.453620Z","shell.execute_reply":"2022-07-30T07:53:57.462614Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feedback_model = FeedBackModel(model_name)\nlightning = Classifier(hparams, feedback_model)\n\ncheckpoint_callback = ModelCheckpoint(\n    monitor=\"val_loss\",\n    dirpath=\"./ckpts/\",\n    mode='min',\n    filename='best',\n    save_weights_only=True\n)\n# lr_monitor = LearningRateMonitor(logging_interval='step')\ntrainer = pl.Trainer(\n    gpus=1, \n    max_epochs=hparams[\"epochs\"], \n    precision=16, \n    gradient_clip_val=1.0, \n    val_check_interval=0.5,\n    logger=wandb_logger,\n    callbacks=[checkpoint_callback]\n)","metadata":{"id":"zCJx2w0w1BfG","outputId":"d6a79502-52b2-4c6e-d43d-14fe07bd322c","execution":{"iopub.status.busy":"2022-07-30T07:53:57.464575Z","iopub.execute_input":"2022-07-30T07:53:57.465045Z","iopub.status.idle":"2022-07-30T07:53:59.589203Z","shell.execute_reply.started":"2022-07-30T07:53:57.465010Z","shell.execute_reply":"2022-07-30T07:53:59.588398Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.fit(lightning)","metadata":{"id":"9ruVw65laXH1","outputId":"2c7741da-da00-4f44-ecfa-dd0fc10681eb","execution":{"iopub.status.busy":"2022-07-30T07:53:59.590457Z","iopub.execute_input":"2022-07-30T07:53:59.590878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{"id":"YhQ27SyJaXH1"}},{"cell_type":"code","source":"predictions = trainer.predict(dataloaders=lightning.test_dataloader(), ckpt_path='best')","metadata":{"id":"COWHXsc_WcIc","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nfor batch in predictions:\n  preds.append(batch)\n\npreds = torch.concat(preds)\npreds = preds.type(torch.float32)\npreds = F.softmax(preds, dim=1)\n# preds.shape\nsample = pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\nsample['Adequate'] = preds[:, 0]\nsample['Effective'] = preds[:, 1]\nsample['Ineffective'] = preds[:, 2]\nprint(sample.head())\nsample.to_csv(\"submission.csv\", index=False)","metadata":{"id":"7OkUkXsuYgi0","trusted":true},"execution_count":null,"outputs":[]}]}