{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport math\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\n\nimport torch\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom transformers import AutoModel, AutoConfig, AutoTokenizer, AdamW, DataCollatorWithPadding\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\nfrom torch.cuda.amp import autocast, GradScaler\n# https://github.com/huggingface/transformers/issues/9919\nfrom torch.utils.checkpoint import checkpoint\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import log_loss\n\nimport warnings\nwarnings.simplefilter('ignore')\nfrom tqdm.notebook import tqdm\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nimport pickle # For roberta test","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.192561Z","iopub.status.idle":"2022-07-11T14:44:20.193258Z","shell.execute_reply.started":"2022-07-11T14:44:20.192989Z","shell.execute_reply":"2022-07-11T14:44:20.193014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = '../input/feedback-prize-effectiveness/'","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.194511Z","iopub.status.idle":"2022-07-11T14:44:20.195196Z","shell.execute_reply.started":"2022-07-11T14:44:20.19495Z","shell.execute_reply":"2022-07-11T14:44:20.194974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nDeberta V3 Base
\n
","metadata":{}},{"cell_type":"markdown","source":"# CFG","metadata":{}},{"cell_type":"code","source":"class CFG:\n wandb = False\n apex = True #\n model = '../input/deberta-v3-base/deberta-v3-base'\n fast = True\n seed = 42\n n_splits = 5\n max_len = 512\n dropout = 0.1\n target_size = 3\n print_freq = 50\n min_lr = 1e-6\n scheduler = 'cosine'\n batch_size = 8\n num_workers = 0\n lr = 3e-5\n weigth_decay = 0.01\n epochs = 3\n n_fold = 5\n trn_fold = [0, 1, 2, 3, 4]\n train = True \n num_warmup_steps = 0 #\n num_cycles=0.5 #\n CVs = []\n debug = False\n debug_ver2 = False\n gradient_checkpointing = True\n AMP = False\n freezing = True\n # after_freezed_parameters = []\n \n n_accumulate= 1\n \n\nif CFG.debug:\n CFG.epochs = 2\n CFG.trn_fold = [0, 1]\n CFG.print_freq = 10\n\nif CFG.debug_ver2:\n CFG.epochs = 1\n CFG.trn_fold = [0, 1]","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.19646Z","iopub.status.idle":"2022-07-11T14:44:20.197146Z","shell.execute_reply.started":"2022-07-11T14:44:20.196897Z","shell.execute_reply":"2022-07-11T14:44:20.196921Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Function","metadata":{}},{"cell_type":"code","source":"# Loss Func\ndef criterion(outputs, labels):\n return nn.CrossEntropyLoss()(outputs, labels)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.19841Z","iopub.status.idle":"2022-07-11T14:44:20.199199Z","shell.execute_reply.started":"2022-07-11T14:44:20.198908Z","shell.execute_reply":"2022-07-11T14:44:20.198936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def softmax(z):\n assert len(z.shape) == 2\n s = np.max(z, axis=1)\n s = s[:, np.newaxis] # necessary step to do broadcasting\n e_x = np.exp(z - s)\n div = np.sum(e_x, axis=1)\n div = div[:, np.newaxis] # dito\n return e_x / div","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.200549Z","iopub.status.idle":"2022-07-11T14:44:20.20122Z","shell.execute_reply.started":"2022-07-11T14:44:20.200972Z","shell.execute_reply":"2022-07-11T14:44:20.200996Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def freeze(module):\n \"\"\"\n Freezes module's parameters.\n \"\"\"\n \n for parameter in module.parameters():\n parameter.requires_grad = False\n \ndef get_freezed_parameters(module):\n \"\"\"\n Returns names of freezed parameters of the given module.\n \"\"\"\n \n freezed_parameters = []\n for name, parameter in module.named_parameters():\n if not parameter.requires_grad:\n freezed_parameters.append(name)\n \n return freezed_parameters","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.202486Z","iopub.status.idle":"2022-07-11T14:44:20.203171Z","shell.execute_reply.started":"2022-07-11T14:44:20.202919Z","shell.execute_reply":"2022-07-11T14:44:20.202943Z"},"_kg_hide-input":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# append to train/test.csv\ndef get_essay(essay_id, is_train=True):\n parent_path = INPUT_DIR + 'train' if is_train else INPUT_DIR + 'test' \n essay_path = os.path.join(parent_path, f\"{essay_id}.txt\")\n essay_text = open(essay_path, 'r').read()\n return essay_text","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.20444Z","iopub.status.idle":"2022-07-11T14:44:20.205128Z","shell.execute_reply.started":"2022-07-11T14:44:20.204876Z","shell.execute_reply":"2022-07-11T14:44:20.204899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess & Tokenizer","metadata":{}},{"cell_type":"code","source":"# Testing Data\ntest = pd.read_csv(INPUT_DIR + 'test.csv')\ntest['essay_text'] = test['essay_id'].apply(lambda x: get_essay(x, is_train=False))\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.206589Z","iopub.status.idle":"2022-07-11T14:44:20.207364Z","shell.execute_reply.started":"2022-07-11T14:44:20.207029Z","shell.execute_reply":"2022-07-11T14:44:20.207071Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.fast:\n tokenizer = AutoTokenizer.from_pretrained(CFG.model, use_fast=True)\nelse:\n tokenizer = AutoTokenizer.from_pretrained(CFG.model)\nCFG.tokenizer = tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.208742Z","iopub.status.idle":"2022-07-11T14:44:20.209428Z","shell.execute_reply.started":"2022-07-11T14:44:20.209161Z","shell.execute_reply":"2022-07-11T14:44:20.209185Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalize","metadata":{}},{"cell_type":"code","source":"from text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\ndef replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n# Register the encoding and decoding error handlers for `utf-8` and `cp1252`.\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n \"\"\"Resolve the encoding problems and normalize the abnormal characters.\"\"\"\n text = (\n text.encode(\"raw_unicode_escape\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n )\n text = unidecode(text)\n return text","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.210695Z","iopub.status.idle":"2022-07-11T14:44:20.211386Z","shell.execute_reply.started":"2022-07-11T14:44:20.211119Z","shell.execute_reply":"2022-07-11T14:44:20.211143Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['discourse_text'] = test['discourse_text'].apply(lambda x : resolve_encodings_and_normalize(x))\ntest['essay_text'] = test['essay_text'].apply(lambda x : resolve_encodings_and_normalize(x))\n# Tokenize the test data\ntest['text'] = test['discourse_type'] + ' '+ test['discourse_text'] + '[SEP]' + test['essay_text']\ntest['label'] = np.nan","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.21267Z","iopub.status.idle":"2022-07-11T14:44:20.213363Z","shell.execute_reply.started":"2022-07-11T14:44:20.213097Z","shell.execute_reply":"2022-07-11T14:44:20.21312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# Testing Datasets\nclass TestDataset(Dataset):\n def __init__(self, cfg, df):\n self.cfg = cfg\n self.text = df['text'].values\n\n def __len__(self):\n return len(self.text)\n\n def __getitem__(self, item):\n inputs = self.cfg.tokenizer.encode_plus(\n self.text[item],\n truncation=True,\n add_special_tokens=True,\n max_length=self.cfg.max_len\n )\n samples = {\n 'input_ids': inputs['input_ids'],\n 'attention_mask': inputs['attention_mask'],\n }\n\n if 'token_type_ids' in inputs:\n samples['token_type_ids'] = inputs['token_type_ids']\n \n return samples","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.214633Z","iopub.status.idle":"2022-07-11T14:44:20.21533Z","shell.execute_reply.started":"2022-07-11T14:44:20.215059Z","shell.execute_reply":"2022-07-11T14:44:20.215083Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dynamic padding","metadata":{}},{"cell_type":"code","source":"# Dynamic Padding (Collate)\n# collate_fn = DataCollatorWithPadding(tokenizer=CFG.tokenizer)\nclass Collate:\n def __init__(self, tokenizer, isTrain=True):\n self.tokenizer = tokenizer\n self.isTrain = isTrain\n # self.args = args\n\n def __call__(self, batch):\n output = dict()\n output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n if self.isTrain:\n output[\"target\"] = [sample[\"target\"] for sample in batch]\n\n # calculate max token length of this batch\n batch_max = max([len(ids) for ids in output[\"input_ids\"]])\n\n # add padding\n if self.tokenizer.padding_side == \"right\":\n output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n else:\n output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n\n # convert to tensors\n output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n if self.isTrain:\n output[\"target\"] = torch.tensor(output[\"target\"], dtype=torch.long)\n\n return output","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.216626Z","iopub.status.idle":"2022-07-11T14:44:20.217317Z","shell.execute_reply.started":"2022-07-11T14:44:20.217053Z","shell.execute_reply":"2022-07-11T14:44:20.217077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class MeanPooling(nn.Module):\n def __init__(self):\n super(MeanPooling, self).__init__()\n \n def forward(self, last_hidden_state, attention_mask):\n input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n sum_mask = input_mask_expanded.sum(1)\n sum_mask = torch.clamp(sum_mask, min=1e-9) #\n mean_embeddings = sum_embeddings / sum_mask\n return mean_embeddings","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.218574Z","iopub.status.idle":"2022-07-11T14:44:20.219267Z","shell.execute_reply.started":"2022-07-11T14:44:20.219Z","shell.execute_reply":"2022-07-11T14:44:20.219025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedBackModel(nn.Module):\n def __init__(self, model_name):\n super(FeedBackModel, self).__init__()\n # Header (fast or normal)\n self.model = AutoModel.from_pretrained(model_name)\n \n # Gradient_checkpointing\n if CFG.gradient_checkpointing:\n (self.model).gradient_checkpointing_enable()\n \n # Freezing\n if CFG.freezing:\n # freezing embeddings and first 2 layers of encoder\n freeze((self.model).embeddings)\n freeze((self.model).encoder.layer[:2])\n CFG.after_freezed_parameters = filter(lambda parameter: parameter.requires_grad, (self.model).parameters())\n \n self.config = AutoConfig.from_pretrained(model_name)\n self.drop = nn.Dropout(p=CFG.dropout)\n self.pooler = MeanPooling()\n self.fc = nn.Linear(self.config.hidden_size, CFG.target_size)\n \n def forward(self, ids, mask): \n out = self.model(input_ids=ids, \n attention_mask=mask,\n output_hidden_states=False)\n out = self.pooler(out.last_hidden_state, mask)\n out = self.drop(out)\n outputs = self.fc(out)\n return outputs","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.220729Z","iopub.status.idle":"2022-07-11T14:44:20.221423Z","shell.execute_reply.started":"2022-07-11T14:44:20.221155Z","shell.execute_reply":"2022-07-11T14:44:20.221179Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"# predict the test value result\ndef inference_fn(test_loader, model, device):\n preds = []\n model.eval()\n model.to(device)\n tk0 = tqdm(test_loader, total=len(test_loader))\n for data in tk0:\n ids = data['input_ids'].to(device, dtype = torch.long)\n mask = data['attention_mask'].to(device, dtype = torch.long)\n with torch.no_grad():\n y_preds = model(ids, mask)\n y_preds = softmax(y_preds.to('cpu').numpy())\n # y_preds = y_preds.to('cpu').numpy()\n preds.append(y_preds)\n predictions = np.concatenate(preds)\n return predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.222684Z","iopub.status.idle":"2022-07-11T14:44:20.22338Z","shell.execute_reply.started":"2022-07-11T14:44:20.223114Z","shell.execute_reply":"2022-07-11T14:44:20.223138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testDataset = TestDataset(CFG, test)\ntest_loader = DataLoader(testDataset,\n batch_size = CFG.batch_size,\n shuffle=False,\n collate_fn = Collate(CFG.tokenizer, isTrain=False),\n num_workers = CFG.num_workers,\n pin_memory = True,\n drop_last=False)\ndeberta_predictions = []\nfor i in CFG.trn_fold:\n model = FeedBackModel(CFG.model)\n model.load_state_dict(torch.load('../input/dbv3basemodels202279/models-deberta-v3-base-deberta-v3-base_fold' + str(i) +'_best.pth'))\n prediction = inference_fn(test_loader, model, device)\n deberta_predictions.append(prediction)\n torch.cuda.empty_cache()\n gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.224653Z","iopub.status.idle":"2022-07-11T14:44:20.225352Z","shell.execute_reply.started":"2022-07-11T14:44:20.225086Z","shell.execute_reply":"2022-07-11T14:44:20.22511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save the predictions","metadata":{}},{"cell_type":"code","source":"deb_ineffective = []\ndeb_effective = []\ndeb_adequate = []\n\nfor x in deberta_predictions:\n deb_ineffective.append(x[:, 0])\n deb_adequate.append(x[:, 1])\n deb_effective.append(x[:, 2])\n# list -> dataframe\ndeb_ineffective = pd.DataFrame(deb_ineffective).T\ndeb_adequate = pd.DataFrame(deb_adequate).T\ndeb_effective = pd.DataFrame(deb_effective).T","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.226733Z","iopub.status.idle":"2022-07-11T14:44:20.227547Z","shell.execute_reply.started":"2022-07-11T14:44:20.227251Z","shell.execute_reply":"2022-07-11T14:44:20.227303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"
\nRoberta Base + Deberta-Large
\n
","metadata":{}},{"cell_type":"markdown","source":"reference:https://www.kaggle.com/code/renokan/fork-ensemble-deberta-roberta/notebook","metadata":{}},{"cell_type":"markdown","source":"# Pre process","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport glob\n\nfrom text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\n\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.nn import Parameter\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom transformers import AutoModel, AutoTokenizer, AutoConfig\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.228998Z","iopub.status.idle":"2022-07-11T14:44:20.229687Z","shell.execute_reply.started":"2022-07-11T14:44:20.229425Z","shell.execute_reply":"2022-07-11T14:44:20.229449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n text = (\n text.encode(\"raw_unicode_escape\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n )\n \n text = unidecode(text)\n \n return text\n\n\ndef fetch_essay(essay_id: str, txt_dir: str):\n essay_path = os.path.join(COMP_DIR + txt_dir, essay_id + '.txt')\n essay_text = open(essay_path, 'r').read()\n \n return essay_text\n\n\ndef prepare_input(cfg, text, text_2=None):\n inputs = cfg.tokenizer(text, text_2,\n padding=\"max_length\",\n add_special_tokens=True,\n max_length=cfg.max_len,\n truncation=True)\n\n for k, v in inputs.items():\n inputs[k] = torch.tensor(v, dtype=torch.long)\n \n return inputs\n\n\ndef inference_fn(test_loader, model, device):\n preds = []\n model.eval()\n model.to(device)\n tk0 = tqdm(test_loader, total=len(test_loader))\n \n for inputs in tk0:\n for k, v in inputs.items():\n inputs[k] = v.to(device)\n \n with torch.no_grad():\n output = model(inputs)\n \n preds.append(F.softmax(output).to('cpu').numpy())\n\n return np.concatenate(preds) \n\n\ndef show_gradient(df, n_row=None):\n if not n_row:\n n_row = 5\n\n return df.head(n_row) \\\n .assign(all_mean=lambda x: x.mean(axis=1)) \\\n .style.background_gradient(cmap=cm, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.230968Z","iopub.status.idle":"2022-07-11T14:44:20.231665Z","shell.execute_reply.started":"2022-07-11T14:44:20.231402Z","shell.execute_reply":"2022-07-11T14:44:20.231426Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.precision', 4)\ncm = sns.light_palette('green', as_cmap=True)\nprops_param = \"color:white; font-weight:bold; background-color:green;\"\n\nN_ROW = 10\n\nCOMP_DIR = \"../input/feedback-prize-effectiveness/\"\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.232907Z","iopub.status.idle":"2022-07-11T14:44:20.233609Z","shell.execute_reply.started":"2022-07-11T14:44:20.233344Z","shell.execute_reply":"2022-07-11T14:44:20.233367Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = COMP_DIR + \"test.csv\"\nsubmission_path = COMP_DIR + \"sample_submission.csv\"\n\ntest_origin = pd.read_csv(test_path)\nsubmission_origin = pd.read_csv(submission_path)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.234862Z","iopub.status.idle":"2022-07-11T14:44:20.235552Z","shell.execute_reply.started":"2022-07-11T14:44:20.235296Z","shell.execute_reply":"2022-07-11T14:44:20.23532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_origin.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.236794Z","iopub.status.idle":"2022-07-11T14:44:20.237497Z","shell.execute_reply.started":"2022-07-11T14:44:20.237235Z","shell.execute_reply":"2022-07-11T14:44:20.237272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"../input/feedback-prize-effectiveness/train.csv\"\ncols_list = ['essay_id', 'discourse_text']\nidxs_list = [49, 80, 945, 947, 1870]\n\ntemp = pd.read_csv(data_path, usecols=cols_list).loc[idxs_list, :]\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.238741Z","iopub.status.idle":"2022-07-11T14:44:20.239431Z","shell.execute_reply.started":"2022-07-11T14:44:20.239164Z","shell.execute_reply":"2022-07-11T14:44:20.239188Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp['discourse_text_UPD'] = temp['discourse_text'].apply(resolve_encodings_and_normalize)\n\ntemp['essay_text'] = temp['essay_id'].transform(fetch_essay, txt_dir='train')\ntemp['essay_text_UPD'] = temp['essay_text'].apply(resolve_encodings_and_normalize)\n\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.240802Z","iopub.status.idle":"2022-07-11T14:44:20.241489Z","shell.execute_reply.started":"2022-07-11T14:44:20.241225Z","shell.execute_reply":"2022-07-11T14:44:20.241264Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n, row in enumerate(temp.iterrows()):\n indx, data = row\n disc_text = data.discourse_text\n disc_text_upd = data.discourse_text_UPD\n\n print(f'\\nN{n} === index: {indx} ===')\n print(f'\\n>>> origin text:')\n print(repr(disc_text))\n print(f'\\n>>> updated text:')\n print(repr(disc_text_upd))","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.242746Z","iopub.status.idle":"2022-07-11T14:44:20.243433Z","shell.execute_reply.started":"2022-07-11T14:44:20.243169Z","shell.execute_reply":"2022-07-11T14:44:20.243193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deberta Large","metadata":{}},{"cell_type":"code","source":"class TestDataset(Dataset):\n def __init__(self, cfg, df):\n self.cfg = cfg\n self.text = df['text'].values\n\n def __len__(self):\n return len(self.text)\n\n def __getitem__(self, item): \n text = self.text[item]\n inputs = prepare_input(self.cfg, text)\n \n return inputs\n\nclass CustomModel(nn.Module):\n def __init__(self, cfg, config_path=None, pretrained=False):\n super().__init__()\n self.cfg = cfg\n \n if config_path is None:\n self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n else:\n self.config = torch.load(config_path)\n \n if pretrained:\n self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n else:\n self.model = AutoModel.from_config(self.config)\n \n self.bilstm = nn.LSTM(self.config.hidden_size, (self.config.hidden_size) // 2, num_layers=2, \n dropout=self.config.hidden_dropout_prob, batch_first=True,\n bidirectional=True)\n \n # self.dropout = nn.Dropout(0.2)\n self.dropout1 = nn.Dropout(0.1)\n self.dropout2 = nn.Dropout(0.2)\n self.dropout3 = nn.Dropout(0.3)\n self.dropout4 = nn.Dropout(0.4)\n self.dropout5 = nn.Dropout(0.5)\n \n self.output = nn.Sequential(\n nn.Linear(self.config.hidden_size, 3) # self.cfg.target_size\n )\n \n def _init_weights(self, module):\n if isinstance(module, nn.Linear):\n module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n if module.bias is not None:\n module.bias.data.zero_()\n elif isinstance(module, nn.Embedding):\n module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n if module.padding_idx is not None:\n module.weight.data[module.padding_idx].zero_()\n elif isinstance(module, nn.LayerNorm):\n module.bias.data.zero_()\n module.weight.data.fill_(1.0)\n\n def forward(self, inputs):\n sequence_output = self.model(**inputs)[0][:, 0, :]\n\n logits1 = self.output(self.dropout1(sequence_output))\n logits2 = self.output(self.dropout2(sequence_output))\n logits3 = self.output(self.dropout3(sequence_output))\n logits4 = self.output(self.dropout4(sequence_output))\n logits5 = self.output(self.dropout5(sequence_output))\n logits = (logits1 + logits2 + logits3 + logits4 + logits5) / 5\n\n return logits","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.244725Z","iopub.status.idle":"2022-07-11T14:44:20.245423Z","shell.execute_reply.started":"2022-07-11T14:44:20.245151Z","shell.execute_reply":"2022-07-11T14:44:20.245176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n path = \"../input/feedback-deberta-large-051/\"\n config_path = path+'config.pth'\n model = \"microsoft/deberta-large\"\n num_workers = 2\n batch_size = 32\n max_len = 512\n seed = 42\n n_fold = 4\n # trn_fold = [0, 1, 2, 3]\n # fc_dropout = 0.2\n # target_size = 3\n \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.path + 'tokenizer')","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.246678Z","iopub.status.idle":"2022-07-11T14:44:20.247392Z","shell.execute_reply.started":"2022-07-11T14:44:20.247122Z","shell.execute_reply":"2022-07-11T14:44:20.247146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test_origin.copy()\nSEP = CFG.tokenizer.sep_token\n\ndf['discourse_text'] = df['discourse_text'].apply(resolve_encodings_and_normalize)\ndf['essay_text'] = df['essay_id'].transform(fetch_essay, txt_dir='test')\ndf['essay_text'] = df['essay_text'].apply(resolve_encodings_and_normalize)\ndf['text'] = df['discourse_type'] + ' ' + df['discourse_text'] + SEP + df['essay_text']\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.248656Z","iopub.status.idle":"2022-07-11T14:44:20.249357Z","shell.execute_reply.started":"2022-07-11T14:44:20.249086Z","shell.execute_reply":"2022-07-11T14:44:20.249111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(CFG, df)\ntest_loader = DataLoader(test_dataset,\n batch_size=CFG.batch_size,\n shuffle=False,\n num_workers=CFG.num_workers,\n pin_memory=True, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.250611Z","iopub.status.idle":"2022-07-11T14:44:20.251298Z","shell.execute_reply.started":"2022-07-11T14:44:20.251035Z","shell.execute_reply":"2022-07-11T14:44:20.251058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deberta_large_predictions = []\n\nfor fold in range(CFG.n_fold):\n model = CustomModel(CFG, config_path=CFG.config_path, pretrained=False)\n state = torch.load(CFG.path+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\",\n map_location=torch.device('cpu'))\n \n model.load_state_dict(state['model'])\n prediction = inference_fn(test_loader, model, DEVICE)\n \n deberta_large_predictions.append(prediction)\n \n del model, state, prediction; gc.collect()\n torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.252551Z","iopub.status.idle":"2022-07-11T14:44:20.25322Z","shell.execute_reply.started":"2022-07-11T14:44:20.252971Z","shell.execute_reply":"2022-07-11T14:44:20.252995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_large_ineffective = []\ndeb_large_effective = []\ndeb_large_adequate = []\n\nfor x in deberta_large_predictions:\n deb_large_ineffective.append(x[:, 0])\n deb_large_adequate.append(x[:, 1])\n deb_large_effective.append(x[:, 2])\n# list -> dataframe\ndeb_large_ineffective = pd.DataFrame(deb_large_ineffective).T\ndeb_large_adequate = pd.DataFrame(deb_large_adequate).T\ndeb_large_effective = pd.DataFrame(deb_large_effective).T","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.254489Z","iopub.status.idle":"2022-07-11T14:44:20.25517Z","shell.execute_reply.started":"2022-07-11T14:44:20.254919Z","shell.execute_reply":"2022-07-11T14:44:20.254943Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Roberta","metadata":{}},{"cell_type":"code","source":"class TestDataset(Dataset):\n def __init__(self, cfg, df):\n self.cfg = cfg\n self.discourse = df['discourse'].values\n self.essay = df['essay'].values\n \n def __len__(self):\n return len(self.discourse)\n \n def __getitem__(self, item):\n discourse = self.discourse[item]\n essay = self.essay[item]\n \n inputs = prepare_input(self.cfg, discourse, essay)\n \n return inputs\n \nclass FeedBackModel(nn.Module):\n def __init__(self, model_path):\n super(FeedBackModel, self).__init__()\n self.model = AutoModel.from_pretrained(model_path)\n self.linear = nn.Linear(768, 3)\n\n def forward(self, inputs):\n last_hidden_states = self.model(**inputs)[0][:, 0, :]\n outputs = self.linear(last_hidden_states)\n \n return outputs","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.256492Z","iopub.status.idle":"2022-07-11T14:44:20.257303Z","shell.execute_reply.started":"2022-07-11T14:44:20.256994Z","shell.execute_reply":"2022-07-11T14:44:20.25702Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_list = pickle.load(\n open(\"../input/feedback-roberta-ep1/roberta_modellist_ep2.pkl\", \"rb\")\n)\n\nclass CFG:\n path = \"../input/roberta-base/\"\n n_fold = 5\n batch = 16\n max_len = 512\n num_workers = 2\n \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.path)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.258596Z","iopub.status.idle":"2022-07-11T14:44:20.259291Z","shell.execute_reply.started":"2022-07-11T14:44:20.259019Z","shell.execute_reply":"2022-07-11T14:44:20.259043Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test_origin.copy()\n\ntxt_sep = \" \"\ndf['discourse'] = df['discourse_type'].str.lower().str.strip() + txt_sep \\\n + df['discourse_text'].str.lower().str.strip()\n\ndf['essay'] = df['essay_id'].transform(fetch_essay, txt_dir='test').str.lower().str.strip()\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.260552Z","iopub.status.idle":"2022-07-11T14:44:20.261229Z","shell.execute_reply.started":"2022-07-11T14:44:20.260977Z","shell.execute_reply":"2022-07-11T14:44:20.261001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(CFG, df)\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch,\n shuffle=False, num_workers=CFG.num_workers,\n pin_memory=True, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.262478Z","iopub.status.idle":"2022-07-11T14:44:20.263166Z","shell.execute_reply.started":"2022-07-11T14:44:20.262913Z","shell.execute_reply":"2022-07-11T14:44:20.262937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roberta_predicts = []\nfor i in range(CFG.n_fold):\n model = model_list[i]\n \n prediction = inference_fn(test_loader, model, DEVICE)\n roberta_predicts.append(prediction)\n \n del model, prediction\n torch.cuda.empty_cache() \n gc.collect()\n \ndel model_list\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.264413Z","iopub.status.idle":"2022-07-11T14:44:20.265093Z","shell.execute_reply.started":"2022-07-11T14:44:20.26484Z","shell.execute_reply":"2022-07-11T14:44:20.264864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_ineffective = []\nrob_effective = []\nrob_adequate = []\n\nfor x in roberta_predicts:\n rob_ineffective.append(x[:, 0])\n rob_adequate.append(x[:, 1])\n rob_effective.append(x[:, 2])\n\n# list -> dataframe\nrob_ineffective = pd.DataFrame(rob_ineffective).T\nrob_adequate = pd.DataFrame(rob_adequate).T\nrob_effective = pd.DataFrame(rob_effective).T","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.266496Z","iopub.status.idle":"2022-07-11T14:44:20.267331Z","shell.execute_reply.started":"2022-07-11T14:44:20.267017Z","shell.execute_reply":"2022-07-11T14:44:20.267046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Calculate the mean prediction probabilities of each folds\nsubmission = pd.read_csv('../input/feedback-prize-effectiveness/sample_submission.csv')\n\nlevel_names = ['deberta', 'deberta_large', 'roberta']\n\nineffective_ = pd.concat(\n [deb_ineffective, deb_large_ineffective, rob_ineffective],\n keys=level_names, axis=1\n)\n\nadequate_ = pd.concat(\n [deb_adequate,deb_large_adequate, rob_adequate],\n keys=level_names, axis=1\n)\n\neffective_ = pd.concat(\n [deb_effective, deb_large_effective, rob_effective],\n keys=level_names, axis=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.268664Z","iopub.status.idle":"2022-07-11T14:44:20.269359Z","shell.execute_reply.started":"2022-07-11T14:44:20.269087Z","shell.execute_reply":"2022-07-11T14:44:20.269111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_gradient(\n ineffective_,\n N_ROW\n)\nshow_gradient(\n adequate_,\n N_ROW\n)\nshow_gradient(\n effective_,\n N_ROW\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.270617Z","iopub.status.idle":"2022-07-11T14:44:20.271311Z","shell.execute_reply.started":"2022-07-11T14:44:20.271041Z","shell.execute_reply":"2022-07-11T14:44:20.271065Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w_ = [0.25, 0.65, 0.1] # ['deberta_base', 'deberta_large', 'roberta']\nd_ = [('Ineffective', ineffective_),\n ('Adequate', adequate_),\n ('Effective', effective_)]\n\nfor x in d_:\n col_name, df = x\n submission[col_name] = pd.DataFrame(\n {col: df[col].mean(axis=1) for col in level_names}\n ).mul(w_).sum(axis=1) \n\nsubmission.head(N_ROW)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.272558Z","iopub.status.idle":"2022-07-11T14:44:20.273233Z","shell.execute_reply.started":"2022-07-11T14:44:20.272982Z","shell.execute_reply":"2022-07-11T14:44:20.273006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-11T14:44:20.27451Z","iopub.status.idle":"2022-07-11T14:44:20.275197Z","shell.execute_reply.started":"2022-07-11T14:44:20.274947Z","shell.execute_reply":"2022-07-11T14:44:20.274972Z"},"trusted":true},"execution_count":null,"outputs":[]}]}