{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"

Feedback Prize - Predicting Effective Arguments

\n\n\n\n#
Table of Contents
\n
\n[0. Update ](#0)
\n[1. Imports ](#1)
\n[2 Dataset](#2)
\n[3 EDA](#4)
\n[4 Data analysis](#5)
\n[5 Training Config ](#6)
\n[6 Build datasets ](#7)
\n[7 Function ](#8)
\n[8 Build Model ](#9)
\n[9 Start Train ](#10)
","metadata":{}},{"cell_type":"markdown","source":" \n\n

Update :

\n","metadata":{}},{"cell_type":"code","source":"!nvidia-smi","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!pip install --upgrade torch\n!pip install --upgrade wandb\n!pip install --upgrade transformers","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n

Imports :

\n","metadata":{}},{"cell_type":"code","source":"import os \nimport tqdm\nimport copy\nimport time\nimport torch \nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt \n\n\nfrom torch import nn \nfrom torch import optim\nfrom torch.optim import lr_scheduler\nfrom sklearn.preprocessing import LabelEncoder\nfrom IPython.core.display import HTML, display\nfrom torch.utils.data import DataLoader,Dataset\n\n\nfrom transformers import (AdamW, AutoConfig, AutoModel, AutoTokenizer,\n DataCollatorWithPadding,GPT2Model,GPT2LMHeadModel,GPT2PreTrainedModel,GPT2Config)\n#################################\nwarnings.filterwarnings(\"ignore\")\ndevice = torch.device(\"cuda\") if torch.cuda.is_available() else \"cpu\"\nprint('Pytorch Version :{}'.format(torch.__version__))\nprint('CUDA Is available :{}'.format(device))\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n \n

Dataset

\n","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv('../input/feedback-prize-effectiveness/train.csv')\ntest_df = pd.read_csv('../input/feedback-prize-effectiveness/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Train datas\n","metadata":{}},{"cell_type":"code","source":"print(f'train:{train_df.shape}')\ntrain_df.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Test datas","metadata":{}},{"cell_type":"code","source":"print(f\"train: {test_df.shape}\")\ntest_df","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n \n

EDA

\n","metadata":{}},{"cell_type":"code","source":"texts = []\nessay = \"\"\n\n\nfor essay_id in train_df.essay_id.unique()[:10]:\n text = open(f'../input/feedback-prize-effectiveness/train/{essay_id}.txt').read()\n essay += f'{text[:200]}'\n\n#text # SHOW\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":" \ndisplay(HTML(f\"\"\"\n\n \n {essay}\n \n
\n\"\"\"))\n\ndel essay #释放内存\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**论述类型\n每个文章元素都包含话语类型元数据。有7个话语类型值，解释来自数据页。**\n\n* 引子--以统计数字、引文、描述或其他一些手段开始的介绍，以抓住读者的注意力并指向论题\n* 立场--对主要问题的看法或结论\n* 主张--支持该立场的主张\n* 反诉--反驳另一个诉求的诉求，或提出与立场相反的理由\n* 反驳--驳斥反诉的主张\n* 证据--支持主张、反主张或反驳的观点或例子。\n* 结论性声明--重申主张的结论性声明。","metadata":{}},{"cell_type":"markdown","source":"\n\n \n

Data analysis

\n","metadata":{}},{"cell_type":"code","source":"# 设置总体的标签类别\nlabels = ['Adequate', 'Effective', 'Ineffective']\n\n\nfig, axes = plt.subplots(1, 2, sharey=True, figsize=(22, 6))\n# plot Discourse type\nax = axes[0]\nsns.countplot(x=\"discourse_type\", data=train_df, linewidth=1.25, alpha=1, ax=ax, zorder=2)\nax.set_title(\"Discourse type distribution\")\n\n# plot Discourse effectiveness\nax = axes[1]\nsns.countplot(x=\"discourse_effectiveness\", data=train_df, ax=ax)\nax.set_title(\"Discourse Effectiveness distribution\")\n\nfig.show()\n\n\n# 绘制每个话语类型的话语有效性分布图\ndiscourse_types = train_df.discourse_type.unique()\n\n\nfig, axes = plt.subplots(2, 4, sharex='col', sharey='row', figsize=(25, 10))\nfor i, discourse_type in enumerate(discourse_types):\n ax = axes.flatten()[i]\n filtered_df = train_df[train_df.discourse_type == discourse_type]\n sns.countplot(x=\"discourse_effectiveness\", data=filtered_df, ax=ax, order=labels)\n ax.set_title(discourse_type)\n ax.set(xlabel=\"Discourse Effectiveness\", ylabel=None)\n \nfig.delaxes(axes[1,3])\nfig.suptitle('Discourse Effectiveness distribution per Discourse Type', fontsize=15)\nplt.show()\n\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 为话语类型显示例子\ndef show_examples_for_discourse_type(discourse_type):\n filt = train_df[train_df.discourse_type==\"Lead\"].sample(frac=1, random_state=420)\n display(HTML(\n f\"\"\"\n

{discourse_type} examples

\n \n \n \n \n \n \n \n \n \n \n \n
IneffectiveAdequateEffective
{filt[filt.discourse_effectiveness=='Ineffective'].iloc[0].discourse_text}{filt[filt.discourse_effectiveness=='Adequate'].iloc[0].discourse_text}{filt[filt.discourse_effectiveness=='Effective'].iloc[0].discourse_text}
\n \"\"\"\n ))\n \n\n[show_examples_for_discourse_type(dt) for dt in discourse_types];","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n \n

Training Config

\n","metadata":{}},{"cell_type":"code","source":"CONFIG = {\n \"seed\": 69,\n \"epochs\": 3,\n \"model_name\": \"microsoft/deberta-v3-base\",\n \"train_batch_size\": 4,\n \"valid_batch_size\": 16,\n \"max_length\": 512,\n \"learning_rate\": 1e-5,\n \"num_classes\": 3, \n }\n\n# 下载预训练模型\nCONFIG[\"tokenizer\"] = AutoTokenizer.from_pretrained(CONFIG['model_name'])\n\nTRAIN_DIR = \"../input/feedback-prize-effectiveness/train\"\nTEST_DIR = \"../input/feedback-prize-effectiveness/test\"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def set_seed(seed):\n# 为可再生性。\n np.random.seed(seed)\n torch.manual_seed(seed)\n torch.cuda.manual_seed(seed)\n # When running on the CuDNN backend, two further options must be set\n torch.backends.cudnn.deterministic = True\n torch.backends.cudnn.benchmark = False\n # Set a fixed value for the hash seed\n os.environ['PYTHONHASHSEED'] = str(seed)\n \nset_seed(CONFIG['seed'])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n \n

Build Datasets

\n","metadata":{}},{"cell_type":"markdown","source":"## 建立标签的编码器 ","metadata":{}},{"cell_type":"code","source":"encoder = LabelEncoder()\ntrain_df['discourse_effectiveness_label'] = encoder.fit_transform(train_df['discourse_effectiveness']) # 对数据的discourse_effectiveness进行编码\nprint(encoder)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedBackDataset(Dataset):\n def __init__(self, df, tokenizer, max_length, data_path):\n self.df = df # \n self.max_len = max_length # 最大的长度\n self.tokenizer = tokenizer # 拟合器\n self.data_path = data_path # 数据路径\n self.discourse_text = df['discourse_text'].values # 论述文本\n self.targets = df['discourse_effectiveness_label'].values # 标签\n self.essay_id = df['essay_id'].values # 作文id\n \n def __len__(self):\n return len(self.df)\n \n def __getitem__(self, index):\n discourse_text = self.discourse_text[index] #论述\n essay_path = os.path.join(self.data_path, f\"{self.essay_id[index]}.txt\") # 作文的源数据\n essay = open(essay_path, 'r').read() # 读取作文\n text = discourse_text + \" \" + self.tokenizer.sep_token + \" \" + essay\n \n inputs = self.tokenizer.encode_plus(\n text,\n truncation=True,\n add_special_tokens=True,\n max_length=self.max_len\n )\n \n return {\n 'input_ids': inputs['input_ids'], # 输入ids\n 'attention_mask': inputs['attention_mask'], # 注意力掩码\n 'target': self.targets[index] # 标签\n }","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train = train_df.sample(frac=0.8, random_state=42) # 训练数据\ndf_valid = train_df.drop(df_train.index) # 验证数据\n\n\ntrain_dataset = FeedBackDataset(\n df_train, tokenizer=CONFIG[\"tokenizer\"], max_length=10, data_path=TRAIN_DIR)\n\n\nvalid_dataset = FeedBackDataset(\n df_valid, tokenizer=CONFIG[\"tokenizer\"], max_length=CONFIG[\"max_length\"], data_path=TRAIN_DIR)\n\n\ncollate_fn = DataCollatorWithPadding(tokenizer=CONFIG['tokenizer'])\n\n\n\ntrain_loader = DataLoader(\n train_dataset,\n batch_size=CONFIG[\"train_batch_size\"],\n collate_fn=collate_fn,\n num_workers=2,\n shuffle=True,\n pin_memory=True)\n\n\nvalid_loader = DataLoader(\n valid_dataset,\n batch_size=CONFIG[\"valid_batch_size\"],\n collate_fn=collate_fn,\n num_workers=2,\n shuffle=False,\n pin_memory=True)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 数据集的大小","metadata":{}},{"cell_type":"code","source":"df_train.shape, df_valid.shape","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 展示数据的内容","metadata":{}},{"cell_type":"code","source":"for i,j,k in train_loader:\n print(i)\n print(j)\n print(k)\n break","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n \n

Function

\n","metadata":{}},{"cell_type":"code","source":"# 配置每个epochs的流程\ndef train_one_epoch(model, optimizer, dataloader, device):\n model.train()\n # 设置计数器\n total = 0 #总数\n running_loss = 0.0 # 运行的损失 \n correct = 0 #正确率\n\n # 进度条\n bar = tqdm.tqdm(dataloader, total=len(dataloader))\n # 将数据喂入模型\n for data in bar:\n ids = data[\"input_ids\"].to(device, dtype=torch.long) # 文本数据编码\n mask = data[\"attention_mask\"].to(device, dtype=torch.long) # 掩码\n targets = data[\"target\"].to(device, dtype=torch.long) # 标签\n\n batch_size = ids.size(0) # 文本数据的大小\n\n outputs = model(ids, mask) # 模型的输出\n\n loss = criterion(outputs, targets) # 损失\n loss.backward()# 反向传播\n\n optimizer.step() # 执行优化\n optimizer.zero_grad() # 优化器梯度归零\n\n running_loss += loss.item() * batch_size # 运行损失\n total += batch_size # 数据的计数\n\n _, predictions = outputs.max(1) # 输出预测\n correct += (predictions == targets).float().sum().item() # 正确率\n\n epoch_loss = running_loss / total # epochs损失\n acc = correct / total # 准确率\n\n bar.set_postfix(Loss=epoch_loss, Accuracy=acc*100)\n\n return epoch_loss, acc\n\n\n# 模型评估\n@torch.no_grad() # 评估时不计算梯度，也就是不进行反向传播\ndef evaluate(model, dataloader, device):\n model.eval()\n\n total = 0\n running_loss = 0.0\n correct = 0\n\n for data in dataloader:\n ids = data[\"input_ids\"].to(device, dtype=torch.long)\n mask = data[\"attention_mask\"].to(device, dtype=torch.long)\n targets = data[\"target\"].to(device, dtype=torch.long)\n\n batch_size = ids.size(0)\n\n outputs = model(ids, mask)\n\n loss = criterion(outputs, targets)\n\n running_loss += loss.item() * batch_size\n total += batch_size\n \n _, predictions = outputs.max(1)\n correct += (predictions == targets).float().sum().item()\n\n epoch_loss = running_loss / total\n acc = correct / total\n\n print(\"Validation Loss: {:.4f} Accuracy: {:.2f}%\".format(epoch_loss, acc * 100))\n return epoch_loss, acc\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def start_training(model, optimizer, device, num_epochs):\n start = time.time() # 设置开始时间\n best_epoch_loss = np.inf\n \n history = {\"Train Loss\": [], \"Valid Loss\": [], \"Train Acc\": [], \"Valid Acc\": []}\n\n for epoch in range(1, num_epochs + 1):\n print(\"Epoch: \", epoch)\n train_epoch_loss, train_epoch_acc = train_one_epoch(\n model, optimizer, dataloader=train_loader, device=device)\n \n val_epoch_loss, valid_epoch_acc = evaluate(\n model, valid_loader, device=device)\n\n history[\"Train Loss\"].append(train_epoch_loss)\n history[\"Valid Loss\"].append(val_epoch_loss)\n history[\"Train Acc\"].append(train_epoch_acc)\n history[\"Valid Acc\"].append(valid_epoch_acc)\n\n # deep copy the model\n if val_epoch_loss <= best_epoch_loss:\n print(\n f\"Validation Loss Improved ({best_epoch_loss} ---> {val_epoch_loss})\"\n )\n best_epoch_loss = val_epoch_loss\n best_epoch_acc = valid_epoch_acc\n best_model_wts = copy.deepcopy(model.state_dict())\n PATH = f\"best_feedback.bin\"\n torch.save(model.state_dict(), PATH)\n # Save a model file from the current directory\n print(f\"Model Saved\")\n\n print()\n\n end = time.time()\n time_elapsed = end - start\n print(\n \"Training complete in {:.0f}h {:.0f}m {:.0f}s\".format(\n time_elapsed // 3600,\n (time_elapsed % 3600) // 60,\n (time_elapsed % 3600) % 60,\n )\n )\n print(\n \"Best Loss: {:.4f} Best Accuracy: {:.2f}\".format(\n best_epoch_loss, best_epoch_acc * 100\n )\n )\n\n # load best model weights\n model.load_state_dict(best_model_wts)\n\n return model, history\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n \n

Build Model

\n","metadata":{}},{"cell_type":"code","source":"class MeanPooling(nn.Module):\n def __init__(self):\n super(MeanPooling, self).__init__()\n \n def forward(self, last_hidden_state, attention_mask):\n input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n sum_mask = input_mask_expanded.sum(1)\n sum_mask = torch.clamp(sum_mask, min=1e-9)\n mean_embeddings = sum_embeddings / sum_mask\n return mean_embeddings\n \n\nclass FeedBackModel(nn.Module):\n def __init__(self, model_name):\n super(FeedBackModel, self).__init__()\n self.drop = nn.Dropout(p=0.05)\n # 加载预训练模型\n self.model = AutoModel.from_pretrained(model_name)\n self.config = AutoConfig.from_pretrained(model_name)\n \n self.mpool = MeanPooling() # 均值池化\n # 前馈全连接层\n self.fc = nn.Sequential(\n nn.Linear(self.config.hidden_size, CONFIG['num_classes']),\n nn.ReLU(),\n nn.Linear(CONFIG['num_classes'],5000),\n nn.Linear(5000,2000),\n nn.Dropout(0.5),\n nn.Linear(2000,100),\n nn.ReLU(),\n nn.Linear(100,CONFIG['num_classes'])\n \n )\n# self.fc2 = nn.Sequential(\n# nn.Linear(1024, CONFIG['num_classes']),\n# )\n \n def forward(self, ids, mask): \n out = self.model(input_ids=ids,attention_mask=mask,\n output_hidden_states=False)\n out = self.mpool(out.last_hidden_state, mask)\n out = self.drop(out)\n out = self.fc(out)\n# out = self.drop(out)\n# out = self.fc2\n return out","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"\n\n\n \n

Start Train

\n","metadata":{}},{"cell_type":"code","source":"model = FeedBackModel(CONFIG['model_name'])\nmodel.to(device)\noptimizer = AdamW(model.parameters(), lr=CONFIG['learning_rate']) # 选择优化器\ncriterion = nn.CrossEntropyLoss() # 选择损失函数","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model, history = start_training(model, optimizer, device=device, num_epochs=CONFIG['epochs'])","metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"execution_count":null,"outputs":[]}]}