{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Feedback prediction\nkaggle: https://www.kaggle.com/competitions/feedback-prize-effectiveness/overview","metadata":{"id":"0uTJ3UtaaXHl"}},{"cell_type":"markdown","source":"## Import","metadata":{"id":"KaPlO6DhaXHq"}},{"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\n\nimport torch\nfrom torch.utils.data import DataLoader, Dataset\nimport torch.nn.functional as F\nimport torch.optim as optim\nimport torch.nn as nn\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import log_loss\n\nimport pytorch_lightning as pl\nfrom pytorch_lightning.callbacks import ModelCheckpoint, LearningRateMonitor\nfrom transformers import AutoTokenizer, AutoModelForSequenceClassification, AutoConfig, get_cosine_schedule_with_warmup, get_linear_schedule_with_warmup","metadata":{"id":"5MrIl3-eaXHq","execution":{"iopub.status.busy":"2022-07-21T00:34:10.818068Z","iopub.execute_input":"2022-07-21T00:34:10.818476Z","iopub.status.idle":"2022-07-21T00:34:21.089858Z","shell.execute_reply.started":"2022-07-21T00:34:10.818400Z","shell.execute_reply":"2022-07-21T00:34:21.088973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{"id":"hnptXakgaXHs"}},{"cell_type":"code","source":"# train = pd.read_csv('./data/train.csv')\n# test = pd.read_csv('./data/test.csv')\ntrain = pd.read_csv('../input/feedback-prize-effectiveness/train.csv')\ntest = pd.read_csv('../input/feedback-prize-effectiveness/test.csv')\ntrain.shape, test.shape","metadata":{"id":"JFDPaO9DaXHs","outputId":"86404ded-93eb-4b3a-bbbc-27caee60e560","execution":{"iopub.status.busy":"2022-07-21T00:34:21.091460Z","iopub.execute_input":"2022-07-21T00:34:21.092091Z","iopub.status.idle":"2022-07-21T00:34:21.376399Z","shell.execute_reply.started":"2022-07-21T00:34:21.092060Z","shell.execute_reply":"2022-07-21T00:34:21.375529Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head(10)","metadata":{"id":"YyVD5EokaXHt","outputId":"8daa2af8-b5b4-4887-c411-1d478cf9698d","execution":{"iopub.status.busy":"2022-07-21T00:34:21.377728Z","iopub.execute_input":"2022-07-21T00:34:21.378071Z","iopub.status.idle":"2022-07-21T00:34:21.395430Z","shell.execute_reply.started":"2022-07-21T00:34:21.378036Z","shell.execute_reply":"2022-07-21T00:34:21.394720Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test","metadata":{"id":"hMPtnjpvaXHt","outputId":"9184b15a-a0f7-48fa-ae57-ef6e08726014","execution":{"iopub.status.busy":"2022-07-21T00:34:21.397579Z","iopub.execute_input":"2022-07-21T00:34:21.398115Z","iopub.status.idle":"2022-07-21T00:34:21.409773Z","shell.execute_reply.started":"2022-07-21T00:34:21.398080Z","shell.execute_reply":"2022-07-21T00:34:21.408670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"type_values = train['discourse_type'].value_counts()\ntype_values","metadata":{"id":"O1vDF80MaXHu","outputId":"6f83c26c-a013-48e7-e592-f04eb749310d","execution":{"iopub.status.busy":"2022-07-21T00:34:21.410914Z","iopub.execute_input":"2022-07-21T00:34:21.411240Z","iopub.status.idle":"2022-07-21T00:34:21.427681Z","shell.execute_reply.started":"2022-07-21T00:34:21.411206Z","shell.execute_reply":"2022-07-21T00:34:21.426788Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"target_values = train['discourse_effectiveness'].value_counts()\ntarget_values","metadata":{"id":"e0scIeoJaXHu","outputId":"273ece24-22fe-4dc8-82ec-4d232fecbf10","execution":{"iopub.status.busy":"2022-07-21T00:34:21.429187Z","iopub.execute_input":"2022-07-21T00:34:21.429905Z","iopub.status.idle":"2022-07-21T00:34:21.442595Z","shell.execute_reply.started":"2022-07-21T00:34:21.429867Z","shell.execute_reply":"2022-07-21T00:34:21.441945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Preprocess","metadata":{"id":"FdTqUoT0aXHv"}},{"cell_type":"code","source":"train_dir = \"../input/feedback-prize-effectiveness/train\"\ntest_dir = \"../input/feedback-prize-effectiveness/test\"\n# train_dir = './data/train'\n# test_dir = './data/test'\ndef get_essay(eassy_id, dir):\n    essay_dir = os.path.join(dir, f'{eassy_id}.txt')\n    essay_txt = open(essay_dir, 'r').read()\n    return essay_txt\n","metadata":{"id":"I3-e3LQTaXHw","execution":{"iopub.status.busy":"2022-07-21T00:34:21.443569Z","iopub.execute_input":"2022-07-21T00:34:21.443974Z","iopub.status.idle":"2022-07-21T00:34:21.449419Z","shell.execute_reply.started":"2022-07-21T00:34:21.443939Z","shell.execute_reply":"2022-07-21T00:34:21.448453Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train['essay_text'] = train['essay_id'].apply(lambda x: get_essay(x, train_dir))\ntest['essay_text'] = test['essay_id'].apply(lambda x: get_essay(x, test_dir))\ntest.head()","metadata":{"id":"VX0-d6epaXHw","outputId":"65613dc8-68e6-4abf-85cb-c5fc799c7b36","execution":{"iopub.status.busy":"2022-07-21T00:34:21.450780Z","iopub.execute_input":"2022-07-21T00:34:21.451358Z","iopub.status.idle":"2022-07-21T00:34:46.413663Z","shell.execute_reply.started":"2022-07-21T00:34:21.451319Z","shell.execute_reply":"2022-07-21T00:34:46.412922Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ntrain['discourse_effectiveness'] = encoder.fit_transform(train['discourse_effectiveness'])\ntrain['discourse_type'] = encoder.fit_transform(train['discourse_type'])\ntest['discourse_type'] = encoder.fit_transform(test['discourse_type'])","metadata":{"id":"SHP0acMeaXHw","execution":{"iopub.status.busy":"2022-07-21T00:34:46.414926Z","iopub.execute_input":"2022-07-21T00:34:46.415277Z","iopub.status.idle":"2022-07-21T00:34:46.448692Z","shell.execute_reply.started":"2022-07-21T00:34:46.415242Z","shell.execute_reply":"2022-07-21T00:34:46.448011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train.head()\ntest.head()","metadata":{"id":"7H4paH-SaXHx","outputId":"1023331f-7250-4720-95e7-17fb53f58078","execution":{"iopub.status.busy":"2022-07-21T00:34:46.450964Z","iopub.execute_input":"2022-07-21T00:34:46.451302Z","iopub.status.idle":"2022-07-21T00:34:46.465839Z","shell.execute_reply.started":"2022-07-21T00:34:46.451268Z","shell.execute_reply":"2022-07-21T00:34:46.465092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dataset","metadata":{"id":"2e-4-80CaXHx"}},{"cell_type":"code","source":"class EssayDataset(Dataset):\n    def __init__(self, df, tokenizer, max_len=512):\n        self.df = df\n        self.tokenizer = tokenizer\n        self.max_len = max_len\n        self.discourse_type = df['discourse_type'].values\n        self.discourse = df['discourse_text'].values\n        self.essay = df['essay_text'].values\n        if 'discourse_effectiveness' in self.df:\n            self.target = df['discourse_effectiveness'].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        discourse_type = self.discourse_type[idx]\n        discourse = self.discourse[idx]\n        essay = self.essay[idx]\n        text = discourse + \" \" + self.tokenizer.sep_token + \" \" + essay\n\n        encode_dict = self.tokenizer.encode_plus(\n            text,\n            add_special_tokens=True,\n            truncation=True,\n            max_length=self.max_len,\n            padding='max_length',\n            return_attention_mask=True,\n            return_tensors='pt'\n        )\n        ids = encode_dict['input_ids']\n        mask = encode_dict['attention_mask']\n\n        ids = ids.squeeze(0)\n        mask = mask.squeeze(0)\n\n        if 'discourse_effectiveness' in self.df:\n            target = self.target[idx]\n            return {\"ids\" : ids, \"mask\": mask, \"target\": target, \"dense_feature\": discourse_type}\n        return {\"ids\": ids, \"mask\": mask, \"dense_feature\": discourse_type}","metadata":{"id":"v-xj6x7waXHx","execution":{"iopub.status.busy":"2022-07-21T00:34:46.468566Z","iopub.execute_input":"2022-07-21T00:34:46.468829Z","iopub.status.idle":"2022-07-21T00:34:46.480473Z","shell.execute_reply.started":"2022-07-21T00:34:46.468806Z","shell.execute_reply":"2022-07-21T00:34:46.479693Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_name = '../input/deberta-v3-base/deberta-v3-base'\nmodel_name = 'bert-base-uncased'","metadata":{"id":"LeKYJhi9aXHy","execution":{"iopub.status.busy":"2022-07-21T00:35:47.866373Z","iopub.execute_input":"2022-07-21T00:35:47.866957Z","iopub.status.idle":"2022-07-21T00:35:47.871786Z","shell.execute_reply.started":"2022-07-21T00:35:47.866914Z","shell.execute_reply":"2022-07-21T00:35:47.870649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(model_name)\ntokenizer","metadata":{"id":"zRTv9kc-aXHy","outputId":"dd787bbd-b3f6-4db3-ba38-dd310fcfb56e","execution":{"iopub.status.busy":"2022-07-21T00:35:49.583549Z","iopub.execute_input":"2022-07-21T00:35:49.584019Z","iopub.status.idle":"2022-07-21T00:35:54.695375Z","shell.execute_reply.started":"2022-07-21T00:35:49.583983Z","shell.execute_reply":"2022-07-21T00:35:54.694563Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"max_len = 512\ntrain_, valid_ = train_test_split(train, test_size=0.2, random_state=42)\ntrain_dataset = EssayDataset(train_, tokenizer, max_len)\n# train_dataset = EssayDataset(train, tokenizer, max_len)\nvalid_dataset = EssayDataset(valid_, tokenizer, max_len)\ntest_dataset = EssayDataset(test, tokenizer, max_len)","metadata":{"id":"ZyV0Gh4kaXHz","execution":{"iopub.status.busy":"2022-06-04T08:17:12.768452Z","iopub.execute_input":"2022-06-04T08:17:12.768982Z","iopub.status.idle":"2022-06-04T08:17:12.858493Z","shell.execute_reply.started":"2022-06-04T08:17:12.768947Z","shell.execute_reply":"2022-06-04T08:17:12.857336Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{"id":"iOA1SFZ_aXHz"}},{"cell_type":"markdown","source":"Add 'discourse_type' as a feature to the bottom.","metadata":{}},{"cell_type":"code","source":"class FeedBackModel(nn.Module):\n    def __init__(self, model_name):\n        super(FeedBackModel, self).__init__()\n        self.deberta = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=10, output_attentions=False, output_hidden_states=False)\n        self.dropout = nn.Dropout(0.2)\n        self.fc = nn.Linear(10+1, 3)\n        \n    def forward(self, batch):        \n        input_ids, attention_masks, dense_feature = batch['ids'], batch['mask'], batch['dense_feature']\n        out = self.deberta(input_ids, attention_mask=attention_masks, output_hidden_states=False)\n        output = self.dropout(out.logits)\n        dense_feature = dense_feature.reshape((-1, 1))\n        output = torch.concat([out.logits, dense_feature], dim=-1)\n        output = self.fc(output)\n        return output","metadata":{"id":"s_J0KmkZZsci","execution":{"iopub.status.busy":"2022-06-04T08:13:39.291128Z","iopub.execute_input":"2022-06-04T08:13:39.291462Z","iopub.status.idle":"2022-06-04T08:13:39.301084Z","shell.execute_reply.started":"2022-06-04T08:13:39.291434Z","shell.execute_reply":"2022-06-04T08:13:39.299955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = FeedBackModel(model_name)\n# taloader = DataLoader(valid_dataset, batch_size=2, shuffle=True)\n# for batch in teloader:\n#     print(batch)\n#     print(batch['ids'].shape, batch['mask'].shape, batch['target'].shape)\n#     print(batch['target'])\n#     print(model(batch))\n#     break","metadata":{"id":"lafpgY5NaXHz","execution":{"iopub.status.busy":"2022-06-04T08:17:29.215145Z","iopub.execute_input":"2022-06-04T08:17:29.215606Z","iopub.status.idle":"2022-06-04T08:17:34.419579Z","shell.execute_reply.started":"2022-06-04T08:17:29.215572Z","shell.execute_reply":"2022-06-04T08:17:34.418245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Classifier(pl.LightningModule):\n    def __init__(self, hparams, model):\n        super(Classifier, self).__init__()\n        self.save_hyperparameters(ignore=['model'])\n\n        self.model = model\n        self.batch_size = hparams[\"batch_size\"]\n        self.lr = hparams[\"lr\"]\n        self.wd = hparams['weight_decay']\n        self.steps = hparams['total_steps']\n\n\n    def forward(self, batch):\n        output = self.model(batch)\n        return output\n\n    def configure_optimizers(self):\n        optimizer = optim.AdamW(self.model.parameters(), lr=self.lr, weight_decay=self.wd)\n        scheduler = get_cosine_schedule_with_warmup(optimizer, num_warmup_steps=int(self.steps * 0.1), num_training_steps=self.steps)\n        # scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=100, num_training_steps=self.steps)\n        scheduler = {\"scheduler\": scheduler, \"interval\": \"step\", \"frequency\": 1}\n        return [optimizer], [scheduler]\n\n    def training_step(self, batch, batch_idx):\n        output = self.forward(batch)\n        pred_flat = torch.argmax(output, dim=1).flatten()\n        labels_flat = batch['target'].flatten()\n        loss = F.cross_entropy(output, labels_flat)\n        acc = torch.sum(pred_flat == labels_flat) / len(labels_flat)\n        self.log(\"train_loss\", loss, on_epoch=True, on_step=False)\n        self.log(\"train_acc\", acc, on_epoch=True, on_step=False)\n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        output = self.forward(batch)\n        pred_flat = torch.argmax(output, dim=1).flatten()\n        labels_flat = batch['target'].flatten()\n        loss = F.cross_entropy(output, labels_flat)\n        acc = torch.sum(pred_flat == labels_flat) / len(labels_flat)\n        self.log(\"val_loss\", loss)\n        self.log(\"val_acc\", acc)\n#         return {\"val_loss\", loss}\n\n    def train_dataloader(self):\n        train_loader = DataLoader(train_dataset, batch_size=self.batch_size, shuffle=True, num_workers=2, pin_memory=True)\n        return train_loader\n\n    def val_dataloader(self):\n        valid_loader = DataLoader(valid_dataset, batch_size=self.batch_size, shuffle=False, num_workers=2, pin_memory=True)\n        return valid_loader\n\n    def test_dataloader(self):\n        test_loader = DataLoader(test_dataset, batch_size=self.batch_size, shuffle=False, num_workers=2, pin_memory=True)\n        return test_loader","metadata":{"id":"U4K6-Y661BfG","execution":{"iopub.status.busy":"2022-06-04T03:15:14.550905Z","iopub.execute_input":"2022-06-04T03:15:14.551631Z","iopub.status.idle":"2022-06-04T03:15:14.569047Z","shell.execute_reply.started":"2022-06-04T03:15:14.55159Z","shell.execute_reply":"2022-06-04T03:15:14.568243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Training","metadata":{"id":"WyqX08iwaXH0"}},{"cell_type":"code","source":"pl.seed_everything(42)\nhparams = {\n    \"batch_size\": 16,\n    \"lr\": 2e-5,\n    \"weight_decay\": 1e-2,\n    \"epochs\": 3,\n}\nloader = DataLoader(train_dataset, batch_size=hparams[\"batch_size\"])\nhparams[\"total_steps\"] = len(loader) * hparams[\"epochs\"]","metadata":{"id":"2rceJup7aXH0","outputId":"c007fb00-3d7d-4274-c15f-843bc82cbb3d","execution":{"iopub.status.busy":"2022-06-04T03:15:14.570898Z","iopub.execute_input":"2022-06-04T03:15:14.571639Z","iopub.status.idle":"2022-06-04T03:15:14.592523Z","shell.execute_reply.started":"2022-06-04T03:15:14.571605Z","shell.execute_reply":"2022-06-04T03:15:14.591684Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"feedback_model = FeedBackModel(model_name)\nlightning = Classifier(hparams, feedback_model)\n\ncheckpoint_callback = ModelCheckpoint(\n    monitor=\"val_loss\",\n    dirpath=\"./ckpts/\",\n    mode='min',\n    filename='best',\n    save_weights_only=True\n)\n# lr_monitor = LearningRateMonitor(logging_interval='step')\ntrainer = pl.Trainer(\n    gpus=1, \n    max_epochs=hparams[\"epochs\"], \n    precision=16, \n    gradient_clip_val=1.0, \n    val_check_interval=0.5,\n    callbacks=[checkpoint_callback]\n)","metadata":{"id":"zCJx2w0w1BfG","outputId":"d6a79502-52b2-4c6e-d43d-14fe07bd322c","execution":{"iopub.status.busy":"2022-06-04T03:15:14.594019Z","iopub.execute_input":"2022-06-04T03:15:14.594474Z","iopub.status.idle":"2022-06-04T03:15:16.540133Z","shell.execute_reply.started":"2022-06-04T03:15:14.594413Z","shell.execute_reply":"2022-06-04T03:15:16.539316Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"trainer.fit(lightning)","metadata":{"id":"9ruVw65laXH1","outputId":"2c7741da-da00-4f44-ecfa-dd0fc10681eb","execution":{"iopub.status.busy":"2022-06-04T03:15:16.541287Z","iopub.execute_input":"2022-06-04T03:15:16.541742Z","iopub.status.idle":"2022-06-04T03:15:39.982656Z","shell.execute_reply.started":"2022-06-04T03:15:16.541699Z","shell.execute_reply":"2022-06-04T03:15:39.981518Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Inference","metadata":{"id":"YhQ27SyJaXH1"}},{"cell_type":"code","source":"predictions = trainer.predict(dataloaders=lightning.test_dataloader(), ckpt_path='best')","metadata":{"id":"COWHXsc_WcIc","execution":{"iopub.status.busy":"2022-06-04T03:15:39.986847Z","iopub.execute_input":"2022-06-04T03:15:39.994206Z","iopub.status.idle":"2022-06-04T03:15:42.716556Z","shell.execute_reply.started":"2022-06-04T03:15:39.994152Z","shell.execute_reply":"2022-06-04T03:15:42.714581Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = []\nfor batch in predictions:\n  preds.append(batch)\n\npreds = torch.concat(preds)\npreds = preds.type(torch.float32)\npreds = F.softmax(preds, dim=1)\n# preds.shape\nsample = pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\nsample['Adequate'] = preds[:, 0]\nsample['Effective'] = preds[:, 1]\nsample['Ineffective'] = preds[:, 2]\nprint(sample.head())\nsample.to_csv(\"submission.csv\", index=False)","metadata":{"id":"7OkUkXsuYgi0","execution":{"iopub.status.busy":"2022-06-04T03:15:42.717319Z","iopub.status.idle":"2022-06-04T03:15:42.717668Z","shell.execute_reply.started":"2022-06-04T03:15:42.717511Z","shell.execute_reply":"2022-06-04T03:15:42.717527Z"},"trusted":true},"execution_count":null,"outputs":[]}]}