{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**Based on the training notebook:** https://www.kaggle.com/code/raj26000/pytorch-feedback-deberta-base-training","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport os\nimport numpy as np\nimport torch\nfrom torch import nn\nfrom transformers import AutoTokenizer, AutoModel","metadata":{"execution":{"iopub.status.busy":"2022-07-20T15:08:16.363819Z","iopub.execute_input":"2022-07-20T15:08:16.364179Z","iopub.status.idle":"2022-07-20T15:08:16.369019Z","shell.execute_reply.started":"2022-07-20T15:08:16.364152Z","shell.execute_reply":"2022-07-20T15:08:16.368091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CONFIG = {\n    'pretrained_model_path': '../input/debertav3base',\n    'saved_model_checkpoint': '../input/deberta-siamese-dropout04/saved_model_state_deberta_siamese_3-stack_dropout0.4.pt',\n    'device': torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n}","metadata":{"execution":{"iopub.status.busy":"2022-07-20T15:08:16.396670Z","iopub.execute_input":"2022-07-20T15:08:16.397240Z","iopub.status.idle":"2022-07-20T15:08:16.402389Z","shell.execute_reply.started":"2022-07-20T15:08:16.397208Z","shell.execute_reply":"2022-07-20T15:08:16.401248Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fetch_essay_texts(df, train=True):\n    if train:\n        base_path = '../input/feedback-prize-effectiveness/train/'\n    else:\n        base_path = '../input/feedback-prize-effectiveness/test/'\n        \n    essay_texts = {}\n    for filename in os.listdir(base_path):\n        with open(base_path + filename) as f:\n            text = f.readlines()\n            full_text = ' '.join([x for x in text])\n            essay_text = ' '.join([x for x in full_text.split()])\n        essay_texts[filename[:-4]] = essay_text\n    df['essay_text'] = [essay_texts[essay_id] for essay_id in df['essay_id'].values]   \n    return df","metadata":{"execution":{"iopub.status.busy":"2022-07-20T15:08:16.416231Z","iopub.execute_input":"2022-07-20T15:08:16.417097Z","iopub.status.idle":"2022-07-20T15:08:16.426593Z","shell.execute_reply.started":"2022-07-20T15:08:16.417058Z","shell.execute_reply":"2022-07-20T15:08:16.425457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data = pd.read_csv('../input/feedback-prize-effectiveness/test.csv')\ntest_data = fetch_essay_texts(test_data, train=False)\ntokenizer = AutoTokenizer.from_pretrained(CONFIG['pretrained_model_path'])","metadata":{"execution":{"iopub.status.busy":"2022-07-20T15:08:16.431747Z","iopub.execute_input":"2022-07-20T15:08:16.432126Z","iopub.status.idle":"2022-07-20T15:08:17.342953Z","shell.execute_reply.started":"2022-07-20T15:08:16.432094Z","shell.execute_reply":"2022-07-20T15:08:17.341776Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MeanPoolingLayer(nn.Module):\n    def __init__(self):\n        super(MeanPoolingLayer, self).__init__()\n    \n    def forward(self, last_hidden_state, attention_mask):\n        expanded_mask = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        mask_sum = expanded_mask.sum(1)\n        mask_sum = torch.clamp(mask_sum, min=1e-9)\n        masked_hidden_state = torch.sum(last_hidden_state * expanded_mask, 1)\n        return masked_hidden_state / mask_sum","metadata":{"execution":{"iopub.status.busy":"2022-07-20T15:08:17.345283Z","iopub.execute_input":"2022-07-20T15:08:17.345953Z","iopub.status.idle":"2022-07-20T15:08:17.354116Z","shell.execute_reply.started":"2022-07-20T15:08:17.345906Z","shell.execute_reply":"2022-07-20T15:08:17.352813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class DiscourseEffectivenessModel(nn.Module):\n    def __init__(self, num_classes=3):\n        super(DiscourseEffectivenessModel, self).__init__()\n        self.num_classes = num_classes\n        self.pretrained_layer = AutoModel.from_pretrained(CONFIG['pretrained_model_path'])\n        self.pooler = MeanPoolingLayer()\n        self.dropout = nn.Dropout(p=0.4)\n        self.fc = nn.Linear(3*self.pretrained_layer.config.hidden_size, num_classes)\n    \n    def forward(self, discourse_input_ids, discourse_attention_mask, essay_input_ids, essay_attention_mask):\n        discourse_out = self.pretrained_layer(input_ids=discourse_input_ids, attention_mask=discourse_attention_mask)\n        discourse_emb = self.pooler(discourse_out.last_hidden_state, discourse_attention_mask)\n        essay_out = self.pretrained_layer(input_ids=essay_input_ids, attention_mask=essay_attention_mask)\n        essay_emb = self.pooler(essay_out.last_hidden_state, essay_attention_mask)\n        concat_emb = torch.cat([discourse_emb, essay_emb, torch.abs(essay_emb - discourse_emb)], dim=-1)\n        x = self.dropout(concat_emb)\n        x = self.fc(x)\n        return x","metadata":{"execution":{"iopub.status.busy":"2022-07-20T15:08:17.355712Z","iopub.execute_input":"2022-07-20T15:08:17.356146Z","iopub.status.idle":"2022-07-20T15:08:17.367475Z","shell.execute_reply.started":"2022-07-20T15:08:17.356105Z","shell.execute_reply":"2022-07-20T15:08:17.366226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"@torch.no_grad()\ndef inference(essay_text, discourse_type, discourse_text, model):\n    model.eval()\n    input_discourse = discourse_type + ' ' + tokenizer.sep_token + ' ' + discourse_text\n    tokenized_discourse = tokenizer.encode_plus(\n        input_discourse,\n        return_token_type_ids=False,\n        return_attention_mask=True,\n        max_length=512,\n        truncation=True,\n        padding='max_length',\n        add_special_tokens=True,\n        return_tensors='pt'\n    )\n    tokenized_essay = tokenizer.encode_plus(\n                            essay_text,\n                            return_token_type_ids=False,\n                            return_attention_mask=True,\n                            max_length=512,\n                            truncation=True,\n                            padding='max_length',\n                            add_special_tokens=True,\n                            return_tensors='pt',\n                        )\n    discourse_input_ids = tokenized_discourse['input_ids'].to(CONFIG['device'], non_blocking=True)\n    discourse_attention_mask = tokenized_discourse['attention_mask'].to(CONFIG['device'], non_blocking=True)\n    essay_input_ids = tokenized_essay['input_ids'].to(CONFIG['device'], non_blocking=True)\n    essay_attention_mask = tokenized_essay['attention_mask'].to(CONFIG['device'], non_blocking=True)\n    with torch.cuda.amp.autocast():\n        logits = model(discourse_input_ids, discourse_attention_mask, essay_input_ids, essay_attention_mask)\n        probs = nn.Softmax(dim=1)(logits)\n    return probs","metadata":{"execution":{"iopub.status.busy":"2022-07-20T15:08:17.370299Z","iopub.execute_input":"2022-07-20T15:08:17.371312Z","iopub.status.idle":"2022-07-20T15:08:17.381333Z","shell.execute_reply.started":"2022-07-20T15:08:17.371262Z","shell.execute_reply":"2022-07-20T15:08:17.380363Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.DataFrame()\ndf['discourse_id'] = test_data['discourse_id']\nclass0, class1, class2 = [], [], []\nmodel = DiscourseEffectivenessModel().to(CONFIG['device'])\nmodel.load_state_dict(torch.load(CONFIG['saved_model_checkpoint']))\nfor essay_text, discourse_type, discourse_text in zip(test_data['essay_text'].tolist(), test_data['discourse_type'].tolist(), test_data['discourse_text'].tolist()):\n    probs = inference(essay_text, discourse_type, discourse_text, model)\n    class0.append(probs[0][0].item())\n    class1.append(probs[0][1].item())\n    class2.append(probs[0][2].item())\n    \ndf['Ineffective'] = class2\ndf['Adequate'] = class1\ndf['Effective'] = class0\ndf.to_csv('submission.csv', index=False)\ndf","metadata":{"execution":{"iopub.status.busy":"2022-07-20T15:08:17.382726Z","iopub.execute_input":"2022-07-20T15:08:17.383746Z","iopub.status.idle":"2022-07-20T15:08:41.655055Z","shell.execute_reply.started":"2022-07-20T15:08:17.383701Z","shell.execute_reply":"2022-07-20T15:08:41.653386Z"},"trusted":true},"execution_count":null,"outputs":[]}]}