{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Libraries**","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport os\nfrom sklearn.preprocessing import LabelEncoder\nfrom torch.utils.data import Dataset\nfrom transformers import AutoTokenizer\nfrom transformers import DataCollatorWithPadding\nfrom transformers import AdamW\nfrom torch.utils.data import DataLoader\nimport torch.nn as nn\nimport transformers\nimport copy\nfrom collections import defaultdict\nimport gc\nfrom tqdm import tqdm\nfrom sklearn.model_selection import GroupKFold, KFold\nimport joblib\nfrom torch.optim import lr_scheduler\nimport time\nimport torch\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Configurations**","metadata":{}},{"cell_type":"code","source":"EPOCHS = 2\nMODEL_NAME = \"../input/bert-base-uncased\"\nTRAIN_BATCH_SIZE = 8\nVALID_BATCH_SIZE = 16\nMAX_LEN = 512\nLEARNING_RATE = 1e-5\nSCHEDULER = 'CosineAnnealingLR'\nMIN_LR = 1e-6\nT_MAX = 500\nWEIGTH_DECAY = 1e-6\nNFOLDS = 5\nNACCUMULATE = 1\nNCLASSES = 3\nTOKENIZER = AutoTokenizer.from_pretrained(MODEL_NAME)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:02.292529Z","iopub.execute_input":"2022-07-14T14:24:02.293166Z","iopub.status.idle":"2022-07-14T14:24:02.485506Z","shell.execute_reply.started":"2022-07-14T14:24:02.293130Z","shell.execute_reply":"2022-07-14T14:24:02.484541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **GPU**","metadata":{}},{"cell_type":"code","source":"import torch\n\nif torch.cuda.is_available():\n    device= torch.device(\"cuda:0\")\nelse:\n    device = \"cpu\"","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:02.486885Z","iopub.execute_input":"2022-07-14T14:24:02.487236Z","iopub.status.idle":"2022-07-14T14:24:02.557738Z","shell.execute_reply.started":"2022-07-14T14:24:02.487201Z","shell.execute_reply":"2022-07-14T14:24:02.556631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Reading Dataset**","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/feedback-prize-effectiveness/train.csv')\n\ndef fetchEssay(essay_id: str):\n    \"\"\"\n    Read the text file of the specific essay_id\n    \"\"\"\n    essay_path = os.path.join('../input/feedback-prize-effectiveness/train/', essay_id + '.txt')\n    essay_text = open(essay_path, 'r').read()\n    return essay_text\n\ntrain['essay_text'] = train['essay_id'].apply(fetchEssay)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:02.561361Z","iopub.execute_input":"2022-07-14T14:24:02.561815Z","iopub.status.idle":"2022-07-14T14:24:29.354752Z","shell.execute_reply.started":"2022-07-14T14:24:02.561785Z","shell.execute_reply":"2022-07-14T14:24:29.352646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ntrain['discourse_effectiveness'] = encoder.fit_transform(train['discourse_effectiveness'])","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:29.355947Z","iopub.execute_input":"2022-07-14T14:24:29.356292Z","iopub.status.idle":"2022-07-14T14:24:29.380610Z","shell.execute_reply.started":"2022-07-14T14:24:29.356257Z","shell.execute_reply":"2022-07-14T14:24:29.379636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Training Dataset**","metadata":{}},{"cell_type":"code","source":"class FeedBackDataset(Dataset):\n    def __init__(self, df, tokenizer, max_length):\n        self.df = df\n        self.max_length = max_length\n        self.tokenizer = tokenizer\n        self.discourse = df['discourse_text'].values\n        self.essay = df['essay_text'].values\n        self.discourse_type = df['discourse_type']\n        self.targets = df['discourse_effectiveness'].values\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        discourse = self.discourse[index]\n        essay = self.essay[index]\n        discourse_type = self.discourse_type[index]\n        text = discourse + \" \" + self.tokenizer.sep_token + \" \" + essay + \" \" + self.tokenizer.sep_token + \" \" + discourse_type\n        inputs = self.tokenizer.encode_plus(\n                        text,\n                        truncation=True,\n                        add_special_tokens=True,\n                        max_length=self.max_length\n                    )\n        \n        return {\n            'input_ids': inputs['input_ids'],\n            'attention_mask': inputs['attention_mask'],\n            'target': self.targets[index]\n        }","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:29.381877Z","iopub.execute_input":"2022-07-14T14:24:29.382444Z","iopub.status.idle":"2022-07-14T14:24:29.406285Z","shell.execute_reply.started":"2022-07-14T14:24:29.382410Z","shell.execute_reply":"2022-07-14T14:24:29.404931Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"collate_fn = DataCollatorWithPadding(tokenizer=TOKENIZER)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:29.408127Z","iopub.execute_input":"2022-07-14T14:24:29.408887Z","iopub.status.idle":"2022-07-14T14:24:29.419723Z","shell.execute_reply.started":"2022-07-14T14:24:29.408849Z","shell.execute_reply":"2022-07-14T14:24:29.418717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Training Model**","metadata":{}},{"cell_type":"code","source":"\nclass FeedBackModel(nn.Module):\n    def __init__(self, model_name):\n        super(FeedBackModel, self).__init__()\n        self.bert = transformers.BertModel.from_pretrained('../input/bert-base-uncased')\n        self.bert_drop = nn.Dropout(0.4)\n        self.out= nn.Linear(768, 3)\n    \n    def forward(self, ids, mask):\n        sequence_output = self.bert(\n            ids, \n            attention_mask=mask\n        )[0]\n        pooled_output = self.bert(\n            ids, \n            attention_mask=mask\n        )[1]\n        \n        bertOut = self.bert_drop(pooled_output)\n        output = self.out(bertOut)\n        \n        return output","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:25:58.007721Z","iopub.execute_input":"2022-07-14T14:25:58.008086Z","iopub.status.idle":"2022-07-14T14:25:58.015597Z","shell.execute_reply.started":"2022-07-14T14:25:58.008058Z","shell.execute_reply":"2022-07-14T14:25:58.014612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Loss Function**","metadata":{}},{"cell_type":"code","source":"def criterion(outputs, labels):\n    return nn.CrossEntropyLoss()(outputs, labels)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:25:58.434295Z","iopub.execute_input":"2022-07-14T14:25:58.434676Z","iopub.status.idle":"2022-07-14T14:25:58.439157Z","shell.execute_reply.started":"2022-07-14T14:25:58.434645Z","shell.execute_reply":"2022-07-14T14:25:58.438191Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Training Function**","metadata":{}},{"cell_type":"code","source":"def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n    model.train()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:\n        ids = data['input_ids'].to(device, dtype = torch.long)\n        mask = data['attention_mask'].to(device, dtype = torch.long)\n        targets = data['target'].to(device, dtype=torch.long)\n        \n        batch_size = ids.size(0)\n\n        outputs = model(ids, mask)\n        \n        loss = criterion(outputs, targets)\n        loss = loss / NACCUMULATE\n        loss.backward()\n    \n        if (step + 1) % NACCUMULATE == 0:\n            optimizer.step()\n\n            # zero the parameter gradients\n            optimizer.zero_grad()\n\n            if scheduler is not None:\n                scheduler.step()\n                \n        running_loss += (loss.item() * batch_size)\n        dataset_size += batch_size\n        \n        epoch_loss = running_loss / dataset_size\n        \n        bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss,\n                        LR=optimizer.param_groups[0]['lr'])\n    gc.collect()\n    \n    return epoch_loss","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:25:59.027781Z","iopub.execute_input":"2022-07-14T14:25:59.028687Z","iopub.status.idle":"2022-07-14T14:25:59.040592Z","shell.execute_reply.started":"2022-07-14T14:25:59.028639Z","shell.execute_reply":"2022-07-14T14:25:59.039608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Run Training**","metadata":{}},{"cell_type":"code","source":"def run_training(model, optimizer, scheduler, device, num_epochs):\n    # To automatically log gradients\n    if torch.cuda.is_available():\n        print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n    \n    start = time.time()\n    best_model_wts = copy.deepcopy(model.state_dict())\n    best_epoch_loss = np.inf\n    history = defaultdict(list)\n    \n    for epoch in range(1, num_epochs + 1): \n        gc.collect()\n\n        train_epoch_loss = train_one_epoch(model, optimizer, scheduler, \n                                           dataloader=train_loader, \n                                           device=device, epoch=epoch)\n    \n        history['Train Loss'].append(train_epoch_loss)\n    \n    return model, history","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:25:59.541291Z","iopub.execute_input":"2022-07-14T14:25:59.541656Z","iopub.status.idle":"2022-07-14T14:25:59.549941Z","shell.execute_reply.started":"2022-07-14T14:25:59.541627Z","shell.execute_reply":"2022-07-14T14:25:59.548248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Data Preparation**","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:26:00.053290Z","iopub.execute_input":"2022-07-14T14:26:00.054130Z","iopub.status.idle":"2022-07-14T14:26:00.060467Z","shell.execute_reply.started":"2022-07-14T14:26:00.054084Z","shell.execute_reply":"2022-07-14T14:26:00.059557Z"}}},{"cell_type":"code","source":"def prepare_loaders():\n    df_train = train\n    \n    train_dataset = FeedBackDataset(df_train, tokenizer=TOKENIZER, max_length=MAX_LEN)\n\n    train_loader = DataLoader(train_dataset, batch_size=TRAIN_BATCH_SIZE, collate_fn=collate_fn, \n                              num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n    \n    return train_loader","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:26:00.542407Z","iopub.execute_input":"2022-07-14T14:26:00.543376Z","iopub.status.idle":"2022-07-14T14:26:00.550066Z","shell.execute_reply.started":"2022-07-14T14:26:00.543328Z","shell.execute_reply":"2022-07-14T14:26:00.548838Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_scheduler(optimizer):\n    if SCHEDULER == 'CosineAnnealingLR':\n        scheduler = lr_scheduler.CosineAnnealingLR(optimizer,T_max=T_MAX, \n                                                   eta_min=MIN_LR)\n    elif SCHEDULER == 'CosineAnnealingWarmRestarts':\n        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer,T_0=T_MAX, \n                                                             eta_min=MIN_LR)\n    elif SCHEDULER == None:\n        return None\n        \n    return scheduler","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:26:00.993992Z","iopub.execute_input":"2022-07-14T14:26:00.994637Z","iopub.status.idle":"2022-07-14T14:26:01.000292Z","shell.execute_reply.started":"2022-07-14T14:26:00.994597Z","shell.execute_reply":"2022-07-14T14:26:00.999079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Start Training**","metadata":{}},{"cell_type":"code","source":"# Create Dataloaders\ntrain_loader = prepare_loaders()\nmodel = FeedBackModel(MODEL_NAME)\nmodel.to(device)\n\n# Define Optimizer and Scheduler\noptimizer = AdamW(model.parameters(), lr=LEARNING_RATE, weight_decay=WEIGTH_DECAY)\nscheduler = fetch_scheduler(optimizer)\n\nmodel, history = run_training(model, optimizer, scheduler,\n                              device=device,\n                              num_epochs=EPOCHS)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:26:01.872254Z","iopub.execute_input":"2022-07-14T14:26:01.872910Z","iopub.status.idle":"2022-07-14T14:26:23.541149Z","shell.execute_reply.started":"2022-07-14T14:26:01.872871Z","shell.execute_reply":"2022-07-14T14:26:23.539609Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Test Dataset**","metadata":{}},{"cell_type":"code","source":"TEST = \"../input/feedback-prize-effectiveness/test\"","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.966213Z","iopub.status.idle":"2022-07-14T14:24:49.966890Z","shell.execute_reply.started":"2022-07-14T14:24:49.966642Z","shell.execute_reply":"2022-07-14T14:24:49.966666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_essay(essay_id):\n    essay_path = os.path.join(TEST, f\"{essay_id}.txt\")\n    essay_text = open(essay_path, 'r').read()\n    return essay_text","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.968288Z","iopub.status.idle":"2022-07-14T14:24:49.969044Z","shell.execute_reply.started":"2022-07-14T14:24:49.968785Z","shell.execute_reply":"2022-07-14T14:24:49.968808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"../input/feedback-prize-effectiveness/test.csv\")\ndf['essay_text'] = df['essay_id'].apply(get_essay)\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.970289Z","iopub.status.idle":"2022-07-14T14:24:49.970968Z","shell.execute_reply.started":"2022-07-14T14:24:49.970723Z","shell.execute_reply":"2022-07-14T14:24:49.970746Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedBackDataset(Dataset):\n    def __init__(self, df, tokenizer, max_length):\n        self.df = df\n        self.max_len = max_length\n        self.tokenizer = tokenizer\n        self.discourse = df['discourse_text'].values\n        self.essay = df['essay_text'].values\n        self.discourse_type = df['discourse_type']\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        discourse = self.discourse[index]\n        essay = self.essay[index]\n        discourse_type = self.discourse_type[index]\n        text = discourse + \" \" + self.tokenizer.sep_token + \" \" + essay + \" \" + self.tokenizer.sep_token + \" \" + discourse_type\n        inputs = self.tokenizer.encode_plus(\n                        text,\n                        truncation=True,\n                        add_special_tokens=True,\n                        max_length=self.max_len,\n                        padding='max_length'\n                    )\n        \n        ids = inputs['input_ids']\n        mask = inputs['attention_mask']\n        \n        return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long)\n        }","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.972239Z","iopub.status.idle":"2022-07-14T14:24:49.972931Z","shell.execute_reply.started":"2022-07-14T14:24:49.972685Z","shell.execute_reply":"2022-07-14T14:24:49.972708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = FeedBackDataset(df, TOKENIZER, max_length=MAX_LENGTH)\ntest_loader = DataLoader(test_dataset, batch_size=TEST_BATCH_SIZE,\n                         num_workers=2, shuffle=False, pin_memory=True)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.974190Z","iopub.status.idle":"2022-07-14T14:24:49.974972Z","shell.execute_reply.started":"2022-07-14T14:24:49.974663Z","shell.execute_reply":"2022-07-14T14:24:49.974688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch.nn.functional as F\n\n@torch.no_grad()\ndef valid_fn(model, dataloader, device):\n    model.eval()\n    \n    dataset_size = 0\n    running_loss = 0.0\n    \n    PREDS = []\n    \n    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n    for step, data in bar:\n        ids = data['ids'].to(device, dtype = torch.long)\n        mask = data['mask'].to(device, dtype = torch.long)\n        \n        outputs = model(ids, mask)\n        outputs = F.softmax(outputs, dim=1)\n        PREDS.append(outputs.cpu().detach().numpy()) \n    \n    PREDS = np.concatenate(PREDS)\n    gc.collect()\n    \n    return PREDS","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.976446Z","iopub.status.idle":"2022-07-14T14:24:49.977271Z","shell.execute_reply.started":"2022-07-14T14:24:49.976949Z","shell.execute_reply":"2022-07-14T14:24:49.976976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def inference(model_paths, dataloader, device):\n    final_preds = []\n    model = FeedBackModel(MODEL_NAME)\n    model.to(DEVICE)\n        \n    preds = valid_fn(model, dataloader, device)\n    final_preds.append(preds)\n    \n    final_preds = np.array(final_preds)\n    final_preds = np.mean(final_preds, axis=0)\n    return final_preds","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.978530Z","iopub.status.idle":"2022-07-14T14:24:49.979222Z","shell.execute_reply.started":"2022-07-14T14:24:49.978970Z","shell.execute_reply":"2022-07-14T14:24:49.978993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds = inference(model, test_loader, DEVICE)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.980450Z","iopub.status.idle":"2022-07-14T14:24:49.981144Z","shell.execute_reply.started":"2022-07-14T14:24:49.980887Z","shell.execute_reply":"2022-07-14T14:24:49.980910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preds","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.982363Z","iopub.status.idle":"2022-07-14T14:24:49.983057Z","shell.execute_reply.started":"2022-07-14T14:24:49.982799Z","shell.execute_reply":"2022-07-14T14:24:49.982823Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample = pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\nsample.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.984290Z","iopub.status.idle":"2022-07-14T14:24:49.985013Z","shell.execute_reply.started":"2022-07-14T14:24:49.984743Z","shell.execute_reply":"2022-07-14T14:24:49.984767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample['Adequate'] = preds[:, 0]\nsample['Effective'] = preds[:, 1]\nsample['Ineffective'] = preds[:, 2]\n\nsample.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.986272Z","iopub.status.idle":"2022-07-14T14:24:49.986946Z","shell.execute_reply.started":"2022-07-14T14:24:49.986700Z","shell.execute_reply":"2022-07-14T14:24:49.986723Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-14T14:24:49.988181Z","iopub.status.idle":"2022-07-14T14:24:49.988854Z","shell.execute_reply.started":"2022-07-14T14:24:49.988612Z","shell.execute_reply":"2022-07-14T14:24:49.988636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"References:\n\nhttps://www.kaggle.com/code/debarshichanda/pytorch-feedback-deberta-v3-baseline\nhttps://www.kaggle.com/code/debarshichanda/feedback-inference","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}