{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"\n> **Credits:** Based on [this](https://www.kaggle.com/code/arvissu/roberta-base-training-notebook-1-epoch) amazing notebook.\n\n
\n
\n \n
\n I fixed it.\n
\n

Training RoBERTa-Large.

\n
\n\n\n\n

\n_____\n## [Inference Notebook](https://www.kaggle.com/code/thedevastator/roberta-large-deberta-large-lb-small)\n_____\n","metadata":{}},{"cell_type":"markdown","source":"### Imports","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nimport sys\nimport time\nimport torch\nimport pickle\nimport random\nimport numpy as np\nimport transformers\nimport pandas as pd\nimport torch.nn as nn\nfrom tqdm import tqdm\nimport torch.nn.functional as F\nfrom sklearn.metrics import log_loss\nfrom torch.cuda.amp import GradScaler, autocast\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import StratifiedKFold\nfrom transformers import AutoModel, AutoTokenizer, AdamW, get_linear_schedule_with_warmup\n\nimport warnings; warnings.simplefilter('ignore')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-06-13T04:07:07.658157Z","iopub.execute_input":"2022-06-13T04:07:07.65868Z","iopub.status.idle":"2022-06-13T04:07:15.667996Z","shell.execute_reply.started":"2022-06-13T04:07:07.65861Z","shell.execute_reply":"2022-06-13T04:07:15.667202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Utils Functions","metadata":{}},{"cell_type":"code","source":"def fetchEssay(essay_id: str):\n essay_path = os.path.join('../input/feedback-prize-effectiveness/train/', essay_id + '.txt')\n essay_text = open(essay_path, 'r').read()\n return essay_text\n\ndef seed_everything(seed):\n random.seed(seed)\n os.environ['PYTHONHASHSEED'] = str(seed)\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n torch.backends.cudnn.deterministic = True","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:07:15.670626Z","iopub.execute_input":"2022-06-13T04:07:15.671447Z","iopub.status.idle":"2022-06-13T04:07:15.678449Z","shell.execute_reply.started":"2022-06-13T04:07:15.671407Z","shell.execute_reply":"2022-06-13T04:07:15.677666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Configurations","metadata":{}},{"cell_type":"code","source":"FOLDS = 5\nlr = 2e-5\nEPOCHS = 2\nSEED = 2018\nMAX_LEN = 512\nBATCH_SIZE = 8\naccumulation_steps = 4\nseed_everything(SEED)","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:07:15.679736Z","iopub.execute_input":"2022-06-13T04:07:15.680154Z","iopub.status.idle":"2022-06-13T04:07:15.690379Z","shell.execute_reply.started":"2022-06-13T04:07:15.680105Z","shell.execute_reply":"2022-06-13T04:07:15.689627Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Model Definition\n- RoBERTa-Large","metadata":{}},{"cell_type":"code","source":"class callback:\n def __init__(self):\n self.loss = list()\n self.model = list()\n\n def put(self, model, loss):\n self.loss.append(loss)\n self.model.append(model)\n\n def get_model(self):\n ind = np.argmin(self.loss)\n return self.model[ind]\n\nclass FeedBackModel(nn.Module):\n def __init__(self, model_path):\n super(FeedBackModel, self).__init__()\n self.model = AutoModel.from_pretrained(model_path) \n self.linear = nn.Linear(1024, 3)\n\n def forward(self, ids, mask):\n x = self.model(ids, mask)[0][:, 0, :]\n pred = self.linear(x)\n return pred\n\nclass FeedBackDataset(Dataset):\n def __init__(self, data, model_path, is_test = False):\n self.data = data\n self.is_test = is_test\n self.tokenizer = AutoTokenizer.from_pretrained(model_path)\n\n def __getitem__(self, idx):\n text = self.data['discourse_text'].values[idx] + ' ' + self.tokenizer.sep_token * 2 + ' ' + self.data['essay'].values[idx]\n if not self.is_test:\n target_value = self.data[y_cols].values[idx]\n\n inputs = self.tokenizer.encode_plus(text, None, truncation = True, add_special_tokens = True, max_length = MAX_LEN)['input_ids']\n mask = [1] * len(inputs) + [0] * (MAX_LEN - len(inputs))\n mask = torch.tensor(mask, dtype = torch.long)\n if len(inputs) != MAX_LEN: inputs = inputs + [self.tokenizer.pad_token_id] * (MAX_LEN - len(inputs))\n ids = torch.tensor(inputs, dtype = torch.long)\n\n if self.is_test: return {'ids': ids, 'mask': mask, }\n\n else:\n targets = torch.FloatTensor(target_value)\n return {'ids': ids, 'mask': mask, 'targets': targets}\n\n def __len__(self): return len(self.data)","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:07:15.692959Z","iopub.execute_input":"2022-06-13T04:07:15.693665Z","iopub.status.idle":"2022-06-13T04:07:15.708016Z","shell.execute_reply.started":"2022-06-13T04:07:15.693626Z","shell.execute_reply":"2022-06-13T04:07:15.707276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Reading the Training data","metadata":{}},{"cell_type":"code","source":"df = pd.read_csv(\"../input/feedback-prize-effectiveness/train.csv\")\ndf['essay'] = df['essay_id'].apply(fetchEssay)\nnew_label = {\"Ineffective\": 0, \"Adequate\": 1, \"Effective\": 2}\ndf['discourse_effectiveness'] = df['discourse_effectiveness'].apply(lambda x: new_label[x] )","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:07:15.708892Z","iopub.execute_input":"2022-06-13T04:07:15.711349Z","iopub.status.idle":"2022-06-13T04:07:48.179903Z","shell.execute_reply.started":"2022-06-13T04:07:15.711311Z","shell.execute_reply":"2022-06-13T04:07:48.179137Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_path = '../input/robertalarge/'\ny_cols = ['discourse_effectiveness']","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:07:48.182102Z","iopub.execute_input":"2022-06-13T04:07:48.182606Z","iopub.status.idle":"2022-06-13T04:07:48.18709Z","shell.execute_reply.started":"2022-06-13T04:07:48.182568Z","shell.execute_reply":"2022-06-13T04:07:48.185967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### RoBERTa - Large Training\n- **Note:** This cannot run on kaggle notebooks.. \n- Trained model can be found [here](https://www.kaggle.com/datasets/thedevastator/roberta-large-feedback)","metadata":{}},{"cell_type":"code","source":"def train_model():\n \n for i, (train_idx, valid_idx) in enumerate(StratifiedKFold(n_splits = FOLDS).split(df, y = df['essay_id'])):\n print(f'fold {i+1}')\n gc.collect()\n\n train_loader = torch.utils.data.DataLoader(FeedBackDataset(df.loc[train_idx, :].reset_index(drop=True), model_path), batch_size=BATCH_SIZE, shuffle=True, num_workers=2)\n val_loader = torch.utils.data.DataLoader(FeedBackDataset(df.loc[valid_idx, :].reset_index(drop=True), model_path), batch_size=BATCH_SIZE, shuffle=False, num_workers=2)\n net = FeedBackModel(model_path)\n net.cuda()\n\n loss_fn = torch.nn.CrossEntropyLoss()\n optimizer = AdamW(net.parameters(), lr = lr)\n param_optimizer = list(net.named_parameters())\n no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n optimizer_grouped_parameters = [{'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01}, {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]\n num_train_optimization_steps = int(EPOCHS * len(train_loader) / accumulation_steps)\n scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0.05 * num_train_optimization_steps, num_training_steps = num_train_optimization_steps)\n scaler = torch.cuda.amp.GradScaler()\n\n for epoch in range(EPOCHS):\n start_time = time.time()\n avg_loss = 0.0\n net.train()\n tbar = tqdm(train_loader)\n loss_list = []\n val_loss_list = []\n\n for step, data in enumerate(tbar):\n input_ids = data['ids'].cuda()\n input_masks = data['mask'].cuda()\n targets = data['targets'].long().view(-1).cuda()\n with torch.cuda.amp.autocast():\n pred = net(input_ids,input_masks)\n loss = loss_fn(pred, targets)\n scaler.scale(loss).backward()\n if step % accumulation_steps == 0 or step == len(tbar) - 1:\n scaler.step(optimizer)\n scaler.update()\n optimizer.zero_grad()\n scheduler.step()\n loss_list.append(loss.detach().cpu().item())\n avg_loss = np.round(np.mean(loss_list), 4)\n tbar.set_description(f\"Epoch {epoch + 1} Loss: {avg_loss} lr: {scheduler.get_last_lr()}\")\n net.eval()\n avg_val_loss = 0.0\n tbar_val = tqdm(val_loader)\n for step, data in enumerate(tbar_val):\n input_ids = data['ids'].cuda()\n input_masks = data['mask'].cuda()\n targets = data['targets'].long().view(-1).cuda()\n pred = net(input_ids,input_masks)\n loss = loss_fn(pred, targets)\n val_loss_list.append(loss.detach().cpu().item())\n avg_val_loss = np.round(np.mean(val_loss_list), 4)\n tbar_val.set_description(f\"Epoch {epoch + 1} Loss: {avg_val_loss}\") \n with open('roberta_large_epoch_%s_fold_%s.pkl' % (epoch, i), 'wb') as f: pkl.dump(net.to(torch.device(\"cpu\")), f) \n torch.cuda.empty_cache()\n\n# train_model()","metadata":{"execution":{"iopub.status.busy":"2022-06-13T04:07:48.188725Z","iopub.execute_input":"2022-06-13T04:07:48.189112Z"},"trusted":true},"execution_count":null,"outputs":[]}]}