{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Ensemble of models\n\n1. feedback_deberta_large_LB0.619 / Score: 0.619\n> https://www.kaggle.com/code/brandonhu0215/feedback-deberta-large-lb0-619\n\n2. RoBerta-base Inference v2.0 / Score: 0.649\n> https://www.kaggle.com/code/arvissu/roberta-base-inference-v2-0\n\n#### If this notebook is helpful, please upvote the original versions:\n\n\n## Content\n\n> I tried to get the same result for each model, but by unifying the actions and removing the excess.\n>  \n> For these models, I use the same prepare_input and inference_fn.\n\n```\ndef prepare_input(cfg, text, text_2=None):\n    inputs = cfg.tokenizer(text, text_2,\n                           padding=\"max_length\",\n                           add_special_tokens=True,\n                           max_length=cfg.max_len,\n                           truncation=True)\n\n    [...]\n\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    \n    for inputs in tk0:\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n            \n        with torch.no_grad():\n            output = model(inputs)\n        \n    [...]\n```\n","metadata":{"papermill":{"duration":0.017223,"end_time":"2022-06-29T02:53:47.264285","exception":false,"start_time":"2022-06-29T02:53:47.247062","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 1. Import & Def & Set & Load","metadata":{"papermill":{"duration":0.013915,"end_time":"2022-06-29T02:53:47.292714","exception":false,"start_time":"2022-06-29T02:53:47.278799","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport glob\n\nfrom text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\n\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.nn import Parameter\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom transformers import AutoModel, AutoTokenizer, AutoConfig\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":8.806745,"end_time":"2022-06-29T02:53:56.118166","exception":false,"start_time":"2022-06-29T02:53:47.311421","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:16.261961Z","iopub.execute_input":"2022-07-19T13:13:16.262891Z","iopub.status.idle":"2022-07-19T13:13:24.128136Z","shell.execute_reply.started":"2022-07-19T13:13:16.262798Z","shell.execute_reply":"2022-07-19T13:13:24.127271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    \n    text = unidecode(text)\n    \n    return text\n\n\ndef fetch_essay(essay_id: str, txt_dir: str):\n    essay_path = os.path.join(COMP_DIR + txt_dir, essay_id + '.txt')\n    essay_text = open(essay_path, 'r').read()\n    \n    return essay_text\n\n\ndef prepare_input(cfg, text, text_2=None):\n    inputs = cfg.tokenizer(text, text_2,\n                           padding=\"max_length\",\n                           add_special_tokens=True,\n                           max_length=cfg.max_len,\n                           truncation=True)\n\n    for k, v in inputs.items():\n        inputs[k] = torch.tensor(v, dtype=torch.long)\n        \n    return inputs\n\n\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    \n    for inputs in tk0:\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n            \n        with torch.no_grad():\n            output = model(inputs)\n        \n        preds.append(F.softmax(output).to('cpu').numpy())\n\n    return np.concatenate(preds)  \n\n\ndef show_gradient(df, n_row=None):\n    if not n_row:\n        n_row = 5\n\n    return df.head(n_row) \\\n                .assign(all_mean=lambda x: x.mean(axis=1)) \\\n                    .style.background_gradient(cmap=cm, axis=1)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.02732,"end_time":"2022-06-29T02:53:56.154308","exception":false,"start_time":"2022-06-29T02:53:56.126988","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:24.129997Z","iopub.execute_input":"2022-07-19T13:13:24.130663Z","iopub.status.idle":"2022-07-19T13:13:24.146336Z","shell.execute_reply.started":"2022-07-19T13:13:24.130623Z","shell.execute_reply":"2022-07-19T13:13:24.145493Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.precision', 4)\ncm = sns.light_palette('green', as_cmap=True)\nprops_param = \"color:white; font-weight:bold; background-color:green;\"\n\nN_ROW = 10\n\nCOMP_DIR = \"../input/feedback-prize-effectiveness/\"\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"papermill":{"duration":0.08735,"end_time":"2022-06-29T02:53:56.250659","exception":false,"start_time":"2022-06-29T02:53:56.163309","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:24.148523Z","iopub.execute_input":"2022-07-19T13:13:24.149150Z","iopub.status.idle":"2022-07-19T13:13:24.238531Z","shell.execute_reply.started":"2022-07-19T13:13:24.149112Z","shell.execute_reply":"2022-07-19T13:13:24.237470Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = COMP_DIR + \"test.csv\"\nsubmission_path = COMP_DIR + \"sample_submission.csv\"\n\ntest_origin = pd.read_csv(test_path)\nsubmission_origin = pd.read_csv(submission_path)","metadata":{"papermill":{"duration":0.030936,"end_time":"2022-06-29T02:53:56.290445","exception":false,"start_time":"2022-06-29T02:53:56.259509","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:24.242653Z","iopub.execute_input":"2022-07-19T13:13:24.243406Z","iopub.status.idle":"2022-07-19T13:13:24.264623Z","shell.execute_reply.started":"2022-07-19T13:13:24.243373Z","shell.execute_reply":"2022-07-19T13:13:24.263894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_origin.head()","metadata":{"papermill":{"duration":0.027815,"end_time":"2022-06-29T02:53:56.326953","exception":false,"start_time":"2022-06-29T02:53:56.299138","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:24.265964Z","iopub.execute_input":"2022-07-19T13:13:24.266337Z","iopub.status.idle":"2022-07-19T13:13:24.284456Z","shell.execute_reply.started":"2022-07-19T13:13:24.266301Z","shell.execute_reply":"2022-07-19T13:13:24.283569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Check unidecode(text)\n\n```\ndef resolve_encodings_and_normalize(text: str) -> str:\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    \n    text = unidecode(text)\n```","metadata":{}},{"cell_type":"code","source":"data_path = \"../input/feedback-prize-effectiveness/train.csv\"\ncols_list = ['essay_id', 'discourse_text']\nidxs_list = [49, 80, 945, 947, 1870]\n\ntemp = pd.read_csv(data_path, usecols=cols_list).loc[idxs_list, :]\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:13:24.285753Z","iopub.execute_input":"2022-07-19T13:13:24.286269Z","iopub.status.idle":"2022-07-19T13:13:24.560561Z","shell.execute_reply.started":"2022-07-19T13:13:24.286231Z","shell.execute_reply":"2022-07-19T13:13:24.559764Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp['discourse_text_UPD'] = temp['discourse_text'].apply(resolve_encodings_and_normalize)\n\ntemp['essay_text'] = temp['essay_id'].transform(fetch_essay, txt_dir='train')\ntemp['essay_text_UPD'] = temp['essay_text'].apply(resolve_encodings_and_normalize)\n\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:13:24.561964Z","iopub.execute_input":"2022-07-19T13:13:24.562565Z","iopub.status.idle":"2022-07-19T13:13:24.603518Z","shell.execute_reply.started":"2022-07-19T13:13:24.562526Z","shell.execute_reply":"2022-07-19T13:13:24.602790Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n, row in enumerate(temp.iterrows()):\n    indx, data = row\n    disc_text = data.discourse_text\n    disc_text_upd = data.discourse_text_UPD\n\n    print(f'\\nN{n} === index: {indx} ===')\n    print(f'\\n>>> origin text:')\n    print(repr(disc_text))\n    print(f'\\n>>> updated text:')\n    print(repr(disc_text_upd))","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:13:24.604981Z","iopub.execute_input":"2022-07-19T13:13:24.605342Z","iopub.status.idle":"2022-07-19T13:13:24.612467Z","shell.execute_reply.started":"2022-07-19T13:13:24.605306Z","shell.execute_reply":"2022-07-19T13:13:24.611628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Extract predictions","metadata":{"papermill":{"duration":0.00859,"end_time":"2022-06-29T02:53:56.344023","exception":false,"start_time":"2022-06-29T02:53:56.335433","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 3.1 DeBerta","metadata":{"papermill":{"duration":0.008456,"end_time":"2022-06-29T02:53:56.361487","exception":false,"start_time":"2022-06-29T02:53:56.353031","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.text = df['text'].values\n\n    def __len__(self):\n        return len(self.text)\n\n    def __getitem__(self, item):    \n        text = self.text[item]\n        inputs = prepare_input(self.cfg, text)\n        \n        return inputs\n\nclass CustomModel(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg\n        \n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n        else:\n            self.model = AutoModel.from_config(self.config)\n        \n        self.bilstm = nn.LSTM(self.config.hidden_size, (self.config.hidden_size) // 2, num_layers=2, \n                              dropout=self.config.hidden_dropout_prob, batch_first=True,\n                              bidirectional=True)\n        \n        # self.dropout = nn.Dropout(0.2)\n        self.dropout1 = nn.Dropout(0.1)\n        self.dropout2 = nn.Dropout(0.2)\n        self.dropout3 = nn.Dropout(0.3)\n        self.dropout4 = nn.Dropout(0.6)\n        self.dropout5 = nn.Dropout(0.8)\n        \n        self.output = nn.Sequential(\n            nn.Linear(self.config.hidden_size, 3)  # self.cfg.target_size\n        )\n                \n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.Embedding):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n\n    def forward(self, inputs):\n        sequence_output = self.model(**inputs)[0][:, 0, :]\n\n        logits1 = self.output(self.dropout1(sequence_output))\n        logits2 = self.output(self.dropout2(sequence_output))\n        logits3 = self.output(self.dropout3(sequence_output))\n        logits4 = self.output(self.dropout4(sequence_output))\n        logits5 = self.output(self.dropout5(sequence_output))\n        logits = (logits1 + logits2 + logits3 + logits4 + logits5) / 5\n\n        return logits","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.029514,"end_time":"2022-06-29T02:53:56.401394","exception":false,"start_time":"2022-06-29T02:53:56.37188","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:24.613654Z","iopub.execute_input":"2022-07-19T13:13:24.614586Z","iopub.status.idle":"2022-07-19T13:13:24.635093Z","shell.execute_reply.started":"2022-07-19T13:13:24.614548Z","shell.execute_reply":"2022-07-19T13:13:24.634212Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    path = \"../input/feedback-deberta-large-051/\"\n    config_path = path+'config.pth'\n    model = \"microsoft/deberta-large\"\n    num_workers = 2\n    batch_size = 16\n    max_len = 512\n    seed = 42\n    n_fold = 4\n    # trn_fold = [0, 1, 2, 3]\n    # fc_dropout = 0.2\n    # target_size = 3\n    \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.path + 'tokenizer')","metadata":{"papermill":{"duration":0.182985,"end_time":"2022-06-29T02:53:56.593152","exception":false,"start_time":"2022-06-29T02:53:56.410167","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:24.638831Z","iopub.execute_input":"2022-07-19T13:13:24.639258Z","iopub.status.idle":"2022-07-19T13:13:24.795359Z","shell.execute_reply.started":"2022-07-19T13:13:24.639229Z","shell.execute_reply":"2022-07-19T13:13:24.794473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test_origin.copy()\nSEP = CFG.tokenizer.sep_token\n\ndf['discourse_text'] = df['discourse_text'].apply(resolve_encodings_and_normalize)\ndf['essay_text'] = df['essay_id'].transform(fetch_essay, txt_dir='test')\ndf['essay_text'] = df['essay_text'].apply(resolve_encodings_and_normalize)\ndf['text'] = df['discourse_type'] + ' ' + df['discourse_text'] + SEP + df['essay_text']\n\ndf.head()","metadata":{"papermill":{"duration":0.054662,"end_time":"2022-06-29T02:53:56.657806","exception":false,"start_time":"2022-06-29T02:53:56.603144","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:24.796687Z","iopub.execute_input":"2022-07-19T13:13:24.797523Z","iopub.status.idle":"2022-07-19T13:13:24.831568Z","shell.execute_reply.started":"2022-07-19T13:13:24.797484Z","shell.execute_reply":"2022-07-19T13:13:24.830811Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(CFG, df)\ntest_loader = DataLoader(test_dataset,\n                         batch_size=CFG.batch_size,\n                         shuffle=False,\n                         num_workers=CFG.num_workers,\n                         pin_memory=True, drop_last=False)","metadata":{"papermill":{"duration":0.017801,"end_time":"2022-06-29T02:53:56.685024","exception":false,"start_time":"2022-06-29T02:53:56.667223","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:24.832961Z","iopub.execute_input":"2022-07-19T13:13:24.833340Z","iopub.status.idle":"2022-07-19T13:13:24.838799Z","shell.execute_reply.started":"2022-07-19T13:13:24.833304Z","shell.execute_reply":"2022-07-19T13:13:24.837533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deberta_predictions = []\n\nfor fold in range(CFG.n_fold):\n    model = CustomModel(CFG, config_path=CFG.config_path, pretrained=False)\n    state = torch.load(CFG.path+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\",\n                       map_location=torch.device('cpu'))\n    \n    model.load_state_dict(state['model'])\n    prediction = inference_fn(test_loader, model, DEVICE)\n    \n    deberta_predictions.append(prediction)\n    \n    del model, state, prediction; gc.collect()\n    torch.cuda.empty_cache()","metadata":{"papermill":{"duration":106.455245,"end_time":"2022-06-29T02:55:43.149193","exception":false,"start_time":"2022-06-29T02:53:56.693948","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:13:24.840465Z","iopub.execute_input":"2022-07-19T13:13:24.840934Z","iopub.status.idle":"2022-07-19T13:15:11.289739Z","shell.execute_reply.started":"2022-07-19T13:13:24.840881Z","shell.execute_reply":"2022-07-19T13:15:11.288822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_ineffective = []\ndeb_effective = []\ndeb_adequate = []\n\nfor x in deberta_predictions:\n    deb_ineffective.append(x[:, 0])\n    deb_adequate.append(x[:, 1])\n    deb_effective.append(x[:, 2])","metadata":{"papermill":{"duration":0.018775,"end_time":"2022-06-29T02:55:43.178719","exception":false,"start_time":"2022-06-29T02:55:43.159944","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:11.292122Z","iopub.execute_input":"2022-07-19T13:15:11.292753Z","iopub.status.idle":"2022-07-19T13:15:11.298806Z","shell.execute_reply.started":"2022-07-19T13:15:11.292707Z","shell.execute_reply":"2022-07-19T13:15:11.297983Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_ineffective = pd.DataFrame(deb_ineffective).T\n\nshow_gradient(\n    deb_ineffective,\n    N_ROW)","metadata":{"papermill":{"duration":0.148585,"end_time":"2022-06-29T02:55:43.337415","exception":false,"start_time":"2022-06-29T02:55:43.18883","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:11.300251Z","iopub.execute_input":"2022-07-19T13:15:11.300845Z","iopub.status.idle":"2022-07-19T13:15:11.450644Z","shell.execute_reply.started":"2022-07-19T13:15:11.300806Z","shell.execute_reply":"2022-07-19T13:15:11.449889Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_adequate = pd.DataFrame(deb_adequate).T\n\nshow_gradient(\n    deb_adequate,\n    N_ROW)","metadata":{"papermill":{"duration":0.040998,"end_time":"2022-06-29T02:55:43.389216","exception":false,"start_time":"2022-06-29T02:55:43.348218","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:11.451976Z","iopub.execute_input":"2022-07-19T13:15:11.452829Z","iopub.status.idle":"2022-07-19T13:15:11.480759Z","shell.execute_reply.started":"2022-07-19T13:15:11.452787Z","shell.execute_reply":"2022-07-19T13:15:11.479859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_effective = pd.DataFrame(deb_effective).T\n\nshow_gradient(\n    deb_effective,\n    N_ROW)","metadata":{"papermill":{"duration":0.043433,"end_time":"2022-06-29T02:55:43.443852","exception":false,"start_time":"2022-06-29T02:55:43.400419","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:11.482348Z","iopub.execute_input":"2022-07-19T13:15:11.482977Z","iopub.status.idle":"2022-07-19T13:15:11.516162Z","shell.execute_reply.started":"2022-07-19T13:15:11.482929Z","shell.execute_reply":"2022-07-19T13:15:11.515252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 RoBerta","metadata":{"papermill":{"duration":0.011395,"end_time":"2022-06-29T02:55:43.467154","exception":false,"start_time":"2022-06-29T02:55:43.455759","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.discourse = df['discourse'].values\n        self.essay = df['essay'].values\n        \n    def __len__(self):\n        return len(self.discourse)\n    \n    def __getitem__(self, item):\n        discourse = self.discourse[item]\n        essay = self.essay[item]\n        \n        inputs = prepare_input(self.cfg, discourse, essay)\n        \n        return inputs\n        \nclass FeedBackModel(nn.Module):\n    def __init__(self, model_path):\n        super(FeedBackModel, self).__init__()\n        self.model = AutoModel.from_pretrained(model_path)\n        self.linear = nn.Linear(768, 3)\n\n    def forward(self, inputs):\n        last_hidden_states = self.model(**inputs)[0][:, 0, :]\n        outputs = self.linear(last_hidden_states)\n        \n        return outputs","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.023062,"end_time":"2022-06-29T02:55:43.501615","exception":false,"start_time":"2022-06-29T02:55:43.478553","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:11.517607Z","iopub.execute_input":"2022-07-19T13:15:11.518266Z","iopub.status.idle":"2022-07-19T13:15:11.529127Z","shell.execute_reply.started":"2022-07-19T13:15:11.518228Z","shell.execute_reply":"2022-07-19T13:15:11.528222Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_list = pickle.load(\n    open(\"../input/feedback-roberta-ep1/roberta_modellist_ep2.pkl\", \"rb\")\n)\n\nclass CFG:\n    path = \"../input/roberta-base/\"\n    n_fold = 5\n    batch = 16\n    max_len = 512\n    num_workers = 2\n    \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.path)","metadata":{"papermill":{"duration":23.005949,"end_time":"2022-06-29T02:56:06.518411","exception":false,"start_time":"2022-06-29T02:55:43.512462","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:11.530654Z","iopub.execute_input":"2022-07-19T13:15:11.531405Z","iopub.status.idle":"2022-07-19T13:15:34.918886Z","shell.execute_reply.started":"2022-07-19T13:15:11.531365Z","shell.execute_reply":"2022-07-19T13:15:34.917966Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test_origin.copy()\n\ntxt_sep = \" \"\ndf['discourse'] = df['discourse_type'].str.lower().str.strip() + txt_sep \\\n                + df['discourse_text'].str.lower().str.strip()\n\ndf['essay'] = df['essay_id'].transform(fetch_essay, txt_dir='test').str.lower().str.strip()\ndf.head()","metadata":{"papermill":{"duration":0.042065,"end_time":"2022-06-29T02:56:06.57446","exception":false,"start_time":"2022-06-29T02:56:06.532395","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:34.920135Z","iopub.execute_input":"2022-07-19T13:15:34.920508Z","iopub.status.idle":"2022-07-19T13:15:34.944589Z","shell.execute_reply.started":"2022-07-19T13:15:34.920474Z","shell.execute_reply":"2022-07-19T13:15:34.943810Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(CFG, df)\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch,\n                         shuffle=False, num_workers=CFG.num_workers,\n                         pin_memory=True, drop_last=False)","metadata":{"papermill":{"duration":0.019897,"end_time":"2022-06-29T02:56:06.606774","exception":false,"start_time":"2022-06-29T02:56:06.586877","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:34.945677Z","iopub.execute_input":"2022-07-19T13:15:34.946014Z","iopub.status.idle":"2022-07-19T13:15:34.957833Z","shell.execute_reply.started":"2022-07-19T13:15:34.945979Z","shell.execute_reply":"2022-07-19T13:15:34.956884Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roberta_predicts = []\nfor i in range(CFG.n_fold):\n    model = model_list[i]\n    \n    prediction = inference_fn(test_loader, model, DEVICE)\n    roberta_predicts.append(prediction)\n    \n    del model, prediction\n    torch.cuda.empty_cache()    \n    gc.collect()\n    \ndel model_list\ngc.collect()","metadata":{"papermill":{"duration":3.524253,"end_time":"2022-06-29T02:56:10.143211","exception":false,"start_time":"2022-06-29T02:56:06.618958","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:34.960528Z","iopub.execute_input":"2022-07-19T13:15:34.960796Z","iopub.status.idle":"2022-07-19T13:15:38.618015Z","shell.execute_reply.started":"2022-07-19T13:15:34.960772Z","shell.execute_reply":"2022-07-19T13:15:38.617113Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_ineffective = []\nrob_effective = []\nrob_adequate = []\n\nfor x in roberta_predicts:\n    rob_ineffective.append(x[:, 0])\n    rob_adequate.append(x[:, 1])\n    rob_effective.append(x[:, 2])","metadata":{"papermill":{"duration":0.022375,"end_time":"2022-06-29T02:56:10.178553","exception":false,"start_time":"2022-06-29T02:56:10.156178","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:38.619652Z","iopub.execute_input":"2022-07-19T13:15:38.620067Z","iopub.status.idle":"2022-07-19T13:15:38.625649Z","shell.execute_reply.started":"2022-07-19T13:15:38.620028Z","shell.execute_reply":"2022-07-19T13:15:38.624628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_ineffective = pd.DataFrame(rob_ineffective).T\n\nshow_gradient(\n    rob_ineffective,\n    N_ROW)","metadata":{"papermill":{"duration":0.07251,"end_time":"2022-06-29T02:56:10.263745","exception":false,"start_time":"2022-06-29T02:56:10.191235","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:38.628188Z","iopub.execute_input":"2022-07-19T13:15:38.628904Z","iopub.status.idle":"2022-07-19T13:15:38.664746Z","shell.execute_reply.started":"2022-07-19T13:15:38.628865Z","shell.execute_reply":"2022-07-19T13:15:38.663816Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_adequate = pd.DataFrame(rob_adequate).T\n\nshow_gradient(\n    rob_adequate,\n    N_ROW)","metadata":{"papermill":{"duration":0.095428,"end_time":"2022-06-29T02:56:10.3919","exception":false,"start_time":"2022-06-29T02:56:10.296472","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:38.666228Z","iopub.execute_input":"2022-07-19T13:15:38.666689Z","iopub.status.idle":"2022-07-19T13:15:38.698315Z","shell.execute_reply.started":"2022-07-19T13:15:38.666639Z","shell.execute_reply":"2022-07-19T13:15:38.697588Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_effective = pd.DataFrame(rob_effective).T\n\nshow_gradient(\n    rob_effective,\n    N_ROW)","metadata":{"papermill":{"duration":0.081635,"end_time":"2022-06-29T02:56:10.511529","exception":false,"start_time":"2022-06-29T02:56:10.429894","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:15:38.699557Z","iopub.execute_input":"2022-07-19T13:15:38.700118Z","iopub.status.idle":"2022-07-19T13:15:38.729521Z","shell.execute_reply.started":"2022-07-19T13:15:38.700081Z","shell.execute_reply":"2022-07-19T13:15:38.728795Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import log_loss\n\nimport gensim\nfrom scipy import sparse\nimport lightgbm as lgb\n\nimport warnings\nwarnings.filterwarnings('ignore')\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:15:38.730823Z","iopub.execute_input":"2022-07-19T13:15:38.731403Z","iopub.status.idle":"2022-07-19T13:15:41.231533Z","shell.execute_reply.started":"2022-07-19T13:15:38.731354Z","shell.execute_reply":"2022-07-19T13:15:41.230573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    n_folds = 2\n    \nINPUT_DIR = \"../input/feedback-prize-effectiveness/\"\n\ndef get_train_essay(essay_id):\n    essay_path = os.path.join(INPUT_DIR,f'train/{essay_id}.txt')\n    essay_text = open(essay_path,'r').read()\n    return essay_text\n\ndef get_test_essay(essay_id):\n    essay_path = os.path.join(INPUT_DIR,f'test/{essay_id}.txt')\n    essay_text = open(essay_path,'r').read()\n    return essay_text\n\ntrain = pd.read_csv(INPUT_DIR+'train.csv')\ntest = pd.read_csv(INPUT_DIR+'test.csv')\ntrain['essay_text'] = train['essay_id'].apply(get_train_essay)\ntest['essay_text'] = test['essay_id'].apply(get_test_essay)\n\ndef set_seed(seed=42):\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nset_seed(CFG.seed)\n\neffectiveness_map = {'Ineffective':0, 'Adequate':1, 'Effective':2}\ntrain['target'] = train['discourse_effectiveness'].map(effectiveness_map)\n\nsgkf = StratifiedGroupKFold(n_splits=CFG.n_folds,shuffle=True,random_state=CFG.seed)\n\nfor fold, (_,val_idx) in enumerate(sgkf.split(X=train, y=train['target'], groups=train.essay_id)):\n    train.loc[val_idx,'kfold'] = fold\n\nword2vec_model = gensim.models.KeyedVectors.load_word2vec_format('../input/google-news/GoogleNews-vectors-negative300.bin', binary=True)\nprint(word2vec_model.vectors.shape)\n\ndef avg_feature_vector(sentence, model, num_features):\n    words = sentence.replace('\\n',\" \").replace(',',' ').replace('.',\" \").split()\n    feature_vec = np.zeros((num_features,),dtype=\"float32\")\n    i=0\n    for word in words:\n        try:\n            feature_vec = np.add(feature_vec, model[word])\n        except KeyError as error:\n            feature_vec \n            i = i + 1\n    if len(words) > 0:\n        feature_vec = np.divide(feature_vec, len(words)- i)\n    return feature_vec\n\nparams = {}\nparams[\"objective\"] = 'multiclass'\nparams['metric'] = 'multi_logloss'\nparams['boosting'] = 'gbdt'\nparams['num_class'] = 3\nparams['is_unbalance'] = True\nparams[\"learning_rate\"] = 0.05\nparams[\"lambda_l2\"] = 0.0256\nparams[\"num_leaves\"] = 52\nparams[\"max_depth\"] = 3\nparams[\"feature_fraction\"] = 0.503\nparams[\"bagging_fraction\"] = 0.741\nparams[\"bagging_freq\"] = 8\nparams[\"bagging_seed\"] = 10\nparams[\"min_data_in_leaf\"] = 10\nparams[\"verbosity\"] = -1\nparams[\"random_state\"] = 42\nnum_rounds = 300\n\noof_score = 0\ny_test_pred = np.zeros((test.shape[0], 3))\n\nfor fold in range(CFG.n_folds):\n    print(f'=============fold:{fold}==================')\n    train_fold=train[train['kfold']!=fold].reset_index(drop=True)\n    valid_fold=train[train['kfold']==fold].reset_index(drop=True)\n\n    #word2vec\n\n    #discourse_text\n    word2vec_train_disc_text = np.zeros((len(train_fold.index),300),dtype=\"float32\")\n    word2vec_valid_disc_text = np.zeros((len(valid_fold.index),300),dtype=\"float32\")\n    word2vec_test_disc_text = np.zeros((len(test.index),300),dtype=\"float32\")\n    for i in range(len(train_fold.index)):\n        word2vec_train_disc_text[i] = avg_feature_vector(train_fold[\"discourse_text\"][i], word2vec_model, 300)\n    for i in range(len(valid_fold.index)):\n        word2vec_valid_disc_text[i] = avg_feature_vector(valid_fold[\"discourse_text\"][i], word2vec_model, 300)\n    for i in range(len(test.index)):\n        word2vec_test_disc_text[i] = avg_feature_vector(test[\"discourse_text\"][i], word2vec_model, 300)\n\n    #essay_text\n    word2vec_train_essay_text = np.zeros((len(train_fold.index),300),dtype=\"float32\")\n    word2vec_valid_essay_text = np.zeros((len(valid_fold.index),300),dtype=\"float32\")\n    word2vec_test_essay_text = np.zeros((len(test.index),300),dtype=\"float32\")\n    for i in range(len(train_fold.index)):\n        word2vec_train_essay_text[i] = avg_feature_vector(train_fold[\"essay_text\"][i], word2vec_model, 300)\n    for i in range(len(valid_fold.index)):\n        word2vec_valid_essay_text[i] = avg_feature_vector(valid_fold[\"essay_text\"][i], word2vec_model, 300)\n    for i in range(len(test.index)):\n        word2vec_test_essay_text[i] = avg_feature_vector(test[\"essay_text\"][i], word2vec_model, 300)\n\n    #OneHot\n    ohe = OneHotEncoder()\n    train_type_ohe=sparse.csr_matrix(ohe.fit_transform(train_fold['discourse_type'].values.reshape(-1,1)))\n    valid_type_ohe=sparse.csr_matrix(ohe.transform(valid_fold['discourse_type'].values.reshape(-1,1)))\n    test_type_ohe=sparse.csr_matrix(ohe.transform(test['discourse_type'].values.reshape(-1,1)))\n\n    ohe1 = OneHotEncoder(handle_unknown='ignore')\n    train_type_ohe1=sparse.csr_matrix(ohe1.fit_transform(train_fold['discourse_text'].values.reshape(-1,1)))\n    valid_type_ohe1=sparse.csr_matrix(ohe1.transform(valid_fold['discourse_text'].values.reshape(-1,1)))\n    test_type_ohe1=sparse.csr_matrix(ohe1.transform(test['discourse_text'].values.reshape(-1,1)))\n\n\n    #merge\n    Xtrain_word2vec = sparse.hstack((train_type_ohe,train_type_ohe1,word2vec_train_disc_text,word2vec_train_essay_text))\n    Xvalid_word2vec = sparse.hstack((valid_type_ohe,valid_type_ohe1,word2vec_valid_disc_text,word2vec_valid_essay_text))\n    test_word2vec = sparse.hstack((test_type_ohe,test_type_ohe1,word2vec_test_disc_text,word2vec_test_essay_text))\n\n    #lgbm\n    lgtrain = lgb.Dataset(Xtrain_word2vec, label=train_fold['target'].ravel())\n    lgvalidation = lgb.Dataset(Xvalid_word2vec, label=valid_fold['target'].ravel())\n\n    model = lgb.train(params, lgtrain, num_rounds, \n                    valid_sets=[lgtrain, lgvalidation], \n                    early_stopping_rounds=100, verbose_eval=100)\n\n    y_pred = model.predict(Xvalid_word2vec, num_iteration=model.best_iteration)\n    y_test_pred += model.predict(test_word2vec, num_iteration=model.best_iteration)\n\n    score = log_loss(valid_fold['target'], y_pred)\n    oof_score += score\n\n    print(f'Fold:{fold},valid score:{score}')\n    \ny_test_pred = y_test_pred / float(CFG.n_folds)\noof_score /= float(CFG.n_folds)\nprint(\"Aggregate OOF Score: {}\".format(oof_score))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:15:41.236245Z","iopub.execute_input":"2022-07-19T13:15:41.236545Z","iopub.status.idle":"2022-07-19T13:20:26.979855Z","shell.execute_reply.started":"2022-07-19T13:15:41.236518Z","shell.execute_reply":"2022-07-19T13:20:26.978980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_ineffective = y_test_pred[:,0]\nlgbm_adequate = y_test_pred[:,1]\nlgbm_effective = y_test_pred[:,2]\n\nlgbm_ineffective = pd.DataFrame(lgbm_ineffective)\nlgbm_adequate = pd.DataFrame(lgbm_adequate)\nlgbm_effective = pd.DataFrame(lgbm_effective)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:20:26.981367Z","iopub.execute_input":"2022-07-19T13:20:26.981961Z","iopub.status.idle":"2022-07-19T13:20:26.988148Z","shell.execute_reply.started":"2022-07-19T13:20:26.981903Z","shell.execute_reply":"2022-07-19T13:20:26.987314Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_ineffective","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:20:26.989686Z","iopub.execute_input":"2022-07-19T13:20:26.990163Z","iopub.status.idle":"2022-07-19T13:20:27.005324Z","shell.execute_reply.started":"2022-07-19T13:20:26.990119Z","shell.execute_reply":"2022-07-19T13:20:27.003863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DeBerta Base","metadata":{}},{"cell_type":"code","source":"# import manipulation\nimport numpy as np\nimport pandas as pd\n\n# import Pytorch\nimport torch \nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\n\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.checkpoint import checkpoint\nfrom torch.autograd import Variable\n\n# import wandb\nimport wandb\n\n# import Transformer model\nimport transformers\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig, AdamW\nfrom transformers import DataCollatorWithPadding\nfrom transformers.models.deberta_v2.modeling_deberta_v2 import StableDropout, ContextPooler\n\n# import SKLearn\nfrom sklearn.model_selection import  KFold, GroupKFold, StratifiedKFold, StratifiedGroupKFold\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn.metrics import log_loss\n\n\n# import ...\nimport string\nimport random\nimport os\nimport joblib\nimport gc\nimport copy\nimport time\n\n\n# other\nfrom tqdm import tqdm\nfrom collections import defaultdict\n\nos.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:20:27.006834Z","iopub.execute_input":"2022-07-19T13:20:27.007378Z","iopub.status.idle":"2022-07-19T13:20:28.506610Z","shell.execute_reply.started":"2022-07-19T13:20:27.007338Z","shell.execute_reply":"2022-07-19T13:20:28.505624Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    seed = 2022\n    max_length = 512\n    epoch = 4\n    train_batch_size = 16\n    valid_batch_size = 32\n\n    model_name = \"../input/debertav3base\"\n    token_name = \"../input/debertav3base\"\n\n    scheduler = \"CosineAnnealingLR\"\n    learning_rate = 1e-5\n    min_lr = 1e-6\n    T_max = 500\n    weight_decay = 0.005\n    \n    num_classes = 3\n    n_fold = 3\n    n_accumulate = 2\n    freezing = True\n    gradient_checkpoint = True\n    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n    \n    wandb_id = f\"PL{round(time.time())}\" # ID on WandB\n    group = f'{wandb_id}-Baseline'\n    competition = \"FeedBack\"\n    _wandb_kernel = \"deb\"\n\n    \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.token_name, use_fast=True)\nCFG.tokenizer.model_max_length = CFG.max_length\nCFG.tokenizer.is_fast","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:20:28.508123Z","iopub.execute_input":"2022-07-19T13:20:28.508497Z","iopub.status.idle":"2022-07-19T13:20:29.288275Z","shell.execute_reply.started":"2022-07-19T13:20:28.508453Z","shell.execute_reply":"2022-07-19T13:20:29.287281Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def freeze(module):\n    \"\"\"\n    Freezes module's parameters.\n    \"\"\"\n    for parameter in module.parameters():\n        parameter.requires_grad = False\n        \n        \nclass FeedbackDataset(Dataset):\n    def __init__(self,df, max_length, tokenizer, training=True):\n        self.df = df\n        self.max_len = max_length\n        self.tokenizer = tokenizer\n        self.discourse_type = self.df['discourse_type'].values\n        self.discourse_text = self.df['discourse_text'].values\n        self.essays = self.df['essay_text'].values\n        self.training = training\n        \n        if self.training:\n            self.targets = self.df['discourse_effectiveness'].values\n    \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, index):\n        discourse_type = self.discourse_type[index]\n        discourse_text = self.discourse_text[index]\n        essay = self.essays[index]\n        type_text = discourse_type + ' ' + discourse_text\n        \n        inputs = self.tokenizer.encode_plus(\n            type_text, \n            essay,\n            truncation = True,\n            add_special_tokens = True,\n            return_token_type_ids = True,\n            max_length = self.max_len\n        )\n        \n        samples = {\n            'input_ids': inputs['input_ids'],\n            'attention_mask': inputs['attention_mask'],\n        }\n        \n        if 'token_type_ids' in inputs:\n            samples['token_type_ids'] = inputs['token_type_ids']\n          \n        if self.training:\n            samples['target'] = self.targets[index]\n        \n        return samples\n\n# Dynamic Padding (Collate)\nclass Collate:\n    def __init__(self, tokenizer, isTrain=True):\n        self.tokenizer = tokenizer\n        self.isTrain = isTrain\n        # self.args = args\n\n    def __call__(self, batch):\n        output = dict()\n        output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n        output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n        if self.isTrain:\n            output[\"target\"] = [sample[\"target\"] for sample in batch]\n\n        # calculate max token length of this batch\n        batch_max = max([len(ids) for ids in output[\"input_ids\"]])\n\n        # add padding\n        if self.tokenizer.padding_side == \"right\":\n            output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n        else:\n            output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n\n        # convert to tensors\n        output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n        output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n        if self.isTrain:\n            output[\"target\"] = torch.tensor(output[\"target\"], dtype=torch.long)\n\n        return output\n\n# collate_fn = DataCollatorWithPadding(tokenizer=CFG.tokenizer)\nclass MeanPooling(nn.Module):\n    def __init__(self):\n        super(MeanPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n        sum_mask = input_mask_expanded.sum(1)\n        sum_mask = torch.clamp(sum_mask, min=1e-9)\n        mean_embeddings = sum_embeddings / sum_mask\n\n        return mean_embeddings\n\n    \nclass MeanMaxPooling(nn.Module):\n    def __init__(self):\n        super(MeanMaxPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        mean_pooling_embeddings = torch.mean(last_hidden_state, 1)\n        _, max_pooling_embeddings = torch.max(last_hidden_state, 1)\n        mean_max_embeddings = torch.cat((mean_pooling_embeddings, max_pooling_embeddings), 1)\n        return mean_max_embeddings\n\n    \nclass LSTMPooling(nn.Module):\n    def __init__(self, num_layers, hidden_size, hiddendim_lstm):\n        super(LSTMPooling, self).__init__()\n        self.num_hidden_layers = num_layers\n        self.hidden_size = hidden_size\n        self.hiddendim_lstm = hiddendim_lstm\n        self.lstm = nn.LSTM(self.hidden_size, self.hiddendim_lstm, batch_first=True)\n        self.dropout = nn.Dropout(0.1)\n    \n    def forward(self, all_hidden_states):\n        ## forward\n        hidden_states = torch.stack([all_hidden_states[layer_i][:, 0].squeeze()\n                                     for layer_i in range(1, self.num_hidden_layers+1)], dim=-1)\n        hidden_states = hidden_states.view(-1, self.num_hidden_layers, self.hidden_size)\n        out, _ = self.lstm(hidden_states, None)\n        out = self.dropout(out[:, -1, :])\n        return out\n    \nclass WeightedLayerPooling(nn.Module):\n    def __init__(self, num_hidden_layers, layer_start: int = 4, layer_weights = None):\n        super(WeightedLayerPooling, self).__init__()\n        self.layer_start = layer_start\n        self.num_hidden_layers = num_hidden_layers\n        self.layer_weights = layer_weights if layer_weights is not None \\\n            else nn.Parameter(\n                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float)\n            )\n\n    def forward(self, all_hidden_states):\n        all_layer_embedding = torch.stack(list(all_hidden_states), dim=0)\n        all_layer_embedding = all_layer_embedding[self.layer_start:, :, :, :]\n        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n        return weighted_average\n    \n\nclass MultiSampleDropout(nn.Module):\n    # Multisample Dropout: https://arxiv.org/abs/1905.09788\n    def __init__(self, classifier, start_prob=0.2, num_samples=8, increment=0.01):\n        super(MultiSampleDropout, self).__init__()\n        #self.dropout = nn.Dropout\n        self.dropouts = [StableDropout(start_prob + (increment*i)) for i in range(num_samples)] \n        self.classifier = classifier\n        \n    def forward(self, out):\n        return torch.mean(torch.stack([\n            self.classifier(dropout(out)) for dropout in self.dropouts\n        ], dim=0), dim=0)\n\nclass FeedbackModel(nn.Module):\n    def __init__(self, model_name):\n        super(FeedbackModel, self).__init__()\n        \n        # DeBERTa\n        self.model = AutoModel.from_pretrained(model_name)\n        self.config = AutoConfig.from_pretrained(model_name)\n        \n        # gradient checkpointing\n        if CFG.gradient_checkpoint:\n            self.model.gradient_checkpointing_enable()\n            print(f\"Gradient Checkpointing: {self.model.is_gradient_checkpointing}\")\n\n        # freezing embeddings and first 6 layers of encoder\n        if  CFG.freezing:\n            freeze(self.model.embeddings)\n            freeze(self.model.encoder.layer[:6])\n            \n        # Pooling\n        #self.weighted_pooler = WeightedLayerPooling(num_hidden_layers=self.config.num_hidden_layers, layer_start=4)\n        #self.pooler = MeanPooling()\n        \n        self.context_pooler = ContextPooler(self.config)\n        \n        #self.bilstm = nn.LSTM(self.config.hidden_size, self.config.hidden_size//2, num_layers=2, \n        #                      dropout=self.config.hidden_dropout_prob, batch_first=True,\n        #                      bidirectional=False)\n        \n        #self.drop = nn.Dropout(p=0.2)\n        \n        # Multi Sample Dropout\n        self.fc = nn.Linear(self.config.hidden_size, CFG.num_classes)\n        self.multi_sample_dropout = MultiSampleDropout(self.fc, start_prob=0.2, num_samples=8, increment=0.01)\n\n    def forward(self, ids, mask):        \n        out = self.model(input_ids=ids,attention_mask=mask,\n                        output_hidden_states=True)\n        \n        # out = self.weighted_pooler(out.hidden_states) # For WeightedLayerPooling\n        # out = self.pooler(out, mask) # For MeanPooling\n                \n        #out = self.context_pooler(torch.stack(list(out.hidden_states), dim=0)) # For ContextPooler\n        out = self.context_pooler(out[0]) # For ContextPooler\n\n        # out = self.pooler(out.last_hidden_state, mask)\n\n        outputs = self.multi_sample_dropout(out)\n        \n        #out = self.pooler(out.last_hidden_state, mask)\n        #out = self.bilstm(out)[0]\n        #out = self.drop(out)\n        #outputs = self.fc(out)\n\n        return outputs\n    \n    def set_optimizer_scheduler(self, option=\"Adam8bit\"):\n        if option == \"AdamW\":\n            model_parameters = filter(lambda parameter: parameter.requires_grad, self.parameters())\n\n            # Optimizer and scheduler\n            optimizer = AdamW(model_parameters, lr=CFG.learning_rate, weight_decay = CFG.weight_decay)\n            scheduler = fetch_scheduler(optimizer)\n        elif option == \"Adam8bit\":\n            # Adam 8-bits optimizer\n            no_decay = [\"bias\", \"LayerNorm.weight\"]\n            optimizer_grouped_parameters = [\n                {\n                    \"params\": [p for n, p in self.named_parameters() if not any(nd in n for nd in no_decay) and p[1].requires_grad],\n                    \"weight_decay\": CFG.weight_decay,\n                },\n                {\n                    \"params\": [p for n, p in self.named_parameters() if any(nd in n for nd in no_decay) and p[1].requires_grad],\n                    \"weight_decay\": 0.0,\n                },\n            ]\n\n            # initializing optimizer \n            # bnb_optimizer = bnb.optim.AdamW(params=model_parameters, lr=CFG.learning_rate, weight_decay=CFG.weight_decay, optim_bits=8)\n            optimizer = bnb.optim.Adam8bit(optimizer_grouped_parameters, lr=CFG.learning_rate)\n            print(f\"8-bit Optimizer:\\n\\n{optimizer}\")\n\n            # setting embeddings parameters\n            # set_embedding_parameters_bits(embeddings_path=self.model.embeddings)\n\n            scheduler = fetch_scheduler(optimizer)\n        else:\n            embedding_parameters = filter(lambda parameter: parameter.requires_grad, self.model.parameters())\n\n            optimizer_model = AdamW(embedding_parameters, lr=5e-6, weight_decay = CFG.weight_decay)\n            optimizer_linear = AdamW(model.fc.parameters(), lr=1e-4, weight_decay = CFG.weight_decay)\n\n            scheduler_model = fetch_scheduler(optimizer_model)\n            scheduler_linear = fetch_scheduler(optimizer_linear)\n\n            optimizer = [optimizer_model, optimizer_linear]\n            scheduler = [scheduler_model, scheduler_linear]\n        \n        self.optimizer = optimizer\n        self.scheduler = scheduler\n        return True\n    \n    def get_optimizer_scheduler(self):\n        return self.optimizer, self.scheduler\n    \ntest_df = pd.read_csv(\"../input/feedback-prize-effectiveness/test.csv\")\ntest_df.head()\n\nINPUT_DIR = \"/kaggle/input/feedback-prize-effectiveness\"\nTEST_DIR = os.path.join(INPUT_DIR, \"test\")\nTEST_CSV = os.path.join(INPUT_DIR, \"test.csv\")\n\ndef get_essay_test(essay_id):\n    path = os.path.join(TEST_DIR, f'{essay_id}.txt')\n    essay_text = open(path, 'r').read()\n    return essay_text\n\ntest_df = pd.read_csv(TEST_CSV)\n\ntest_df['essay_text']= test_df['essay_id'].apply(get_essay_test)\ntest_df.head()\n\ncollate_fn = Collate(tokenizer=CFG.tokenizer, isTrain=False)\n\ndef prepare_test_loader(test_df):    \n    test_dataset = FeedbackDataset(test_df, \n                                   tokenizer=CFG.tokenizer, \n                                   max_length=CFG.max_length,\n                                    training=False)\n    \n    test_loader = DataLoader(test_dataset, \n                             batch_size=CFG.valid_batch_size, \n                             collate_fn=collate_fn, \n                             num_workers=2, \n                             shuffle=False, \n                             pin_memory=True, \n                             drop_last=False)\n    return test_loader\n\ntest_loader = prepare_test_loader(test_df)\n\n@torch.no_grad()\ndef inference(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    \n    bar = tqdm(enumerate(test_loader), total=len(test_loader))\n    \n    for step, data in bar: \n        ids = data['input_ids'].to(device, dtype = torch.long)\n        mask = data['attention_mask'].to(device, dtype = torch.long)\n        \n        output = model(ids, mask)\n        y_preds = nn.Softmax(dim=1)(torch.tensor(output.to('cpu'))).numpy()\n        \n        preds.append(y_preds)\n         \n    predictions = np.concatenate(preds)\n    return predictions\n\ndeberta_predictions = []\n\nfor fold in range(CFG.n_fold):\n    print(\"Fold {}\".format(fold))\n\n    model = FeedbackModel(CFG.model_name)\n    state = torch.load(f'../input/deberta-v3-training/LossFold-{fold}.bin')\n\n    model.load_state_dict(state)\n\n    prediction = inference(test_loader, model, CFG.device)\n    deberta_predictions.append(prediction)\n    del model, state, prediction\n    gc.collect()\n    torch.cuda.empty_cache()\n    \ndeberta_base_predictions = np.mean(deberta_predictions, axis=0)\ndeberta_base_predictions\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:20:29.289932Z","iopub.execute_input":"2022-07-19T13:20:29.290436Z","iopub.status.idle":"2022-07-19T13:21:08.129072Z","shell.execute_reply.started":"2022-07-19T13:20:29.290396Z","shell.execute_reply":"2022-07-19T13:21:08.128221Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_base_adequate = deberta_base_predictions[:, 0]\ndeb_base_effective = deberta_base_predictions[:, 1]\ndeb_base_ineffective = deberta_base_predictions[:, 2]\n\ndeb_base_ineffective = pd.DataFrame( deb_base_ineffective)\ndeb_base_adequate = pd.DataFrame( deb_base_adequate)\ndeb_base_effective = pd.DataFrame( deb_base_effective)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-19T13:23:44.162941Z","iopub.execute_input":"2022-07-19T13:23:44.163828Z","iopub.status.idle":"2022-07-19T13:23:44.169479Z","shell.execute_reply.started":"2022-07-19T13:23:44.163785Z","shell.execute_reply":"2022-07-19T13:23:44.168608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Create submission","metadata":{"papermill":{"duration":0.024077,"end_time":"2022-06-29T02:56:10.562061","exception":false,"start_time":"2022-06-29T02:56:10.537984","status":"completed"},"tags":[]}},{"cell_type":"code","source":"level_names = ['deberta_l', 'deberta_b', 'roberta', 'lgbm']\n\nineffective_ = pd.concat(\n    [deb_ineffective, deb_base_ineffective, rob_ineffective, lgbm_ineffective],\n    keys=level_names, axis=1\n)\n\nadequate_ = pd.concat(\n    [deb_adequate, deb_base_adequate, rob_adequate, lgbm_adequate],\n    keys=level_names, axis=1\n)\n\neffective_ = pd.concat(\n    [deb_effective, deb_base_effective, rob_effective, lgbm_effective],\n    keys=level_names, axis=1\n)","metadata":{"papermill":{"duration":0.046338,"end_time":"2022-06-29T02:56:10.633434","exception":false,"start_time":"2022-06-29T02:56:10.587096","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:23:53.310711Z","iopub.execute_input":"2022-07-19T13:23:53.311879Z","iopub.status.idle":"2022-07-19T13:23:53.328688Z","shell.execute_reply.started":"2022-07-19T13:23:53.311835Z","shell.execute_reply":"2022-07-19T13:23:53.327905Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_gradient(\n    ineffective_,\n    N_ROW\n)","metadata":{"papermill":{"duration":0.137257,"end_time":"2022-06-29T02:56:10.79562","exception":false,"start_time":"2022-06-29T02:56:10.658363","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:24:06.741263Z","iopub.execute_input":"2022-07-19T13:24:06.742097Z","iopub.status.idle":"2022-07-19T13:24:06.823476Z","shell.execute_reply.started":"2022-07-19T13:24:06.742060Z","shell.execute_reply":"2022-07-19T13:24:06.822597Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_gradient(\n    adequate_,\n    N_ROW\n)","metadata":{"papermill":{"duration":0.102402,"end_time":"2022-06-29T02:56:10.923467","exception":false,"start_time":"2022-06-29T02:56:10.821065","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:24:12.279217Z","iopub.execute_input":"2022-07-19T13:24:12.279962Z","iopub.status.idle":"2022-07-19T13:24:12.357836Z","shell.execute_reply.started":"2022-07-19T13:24:12.279905Z","shell.execute_reply":"2022-07-19T13:24:12.356978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_gradient(\n    effective_,\n    N_ROW\n)","metadata":{"papermill":{"duration":0.079449,"end_time":"2022-06-29T02:56:11.019182","exception":false,"start_time":"2022-06-29T02:56:10.939733","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:24:12.676539Z","iopub.execute_input":"2022-07-19T13:24:12.677423Z","iopub.status.idle":"2022-07-19T13:24:12.751630Z","shell.execute_reply.started":"2022-07-19T13:24:12.677386Z","shell.execute_reply":"2022-07-19T13:24:12.750812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = submission_origin.copy()\n\nw_ = [.65, .20, .05, .10]  # ['deberta_l', 'deberta_b', 'roberta', 'gbm']\nd_ = [('Ineffective', ineffective_),\n      ('Adequate', adequate_),\n      ('Effective', effective_)]\n\nfor x in d_:\n    col_name, df = x\n    submission[col_name] = pd.DataFrame(\n        {col: df[col].mean(axis=1) for col in level_names}\n    ).mul(w_).sum(axis=1)    \n\nsubmission.head(N_ROW)","metadata":{"papermill":{"duration":0.042962,"end_time":"2022-06-29T02:56:11.078725","exception":false,"start_time":"2022-06-29T02:56:11.035763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:24:13.282672Z","iopub.execute_input":"2022-07-19T13:24:13.283590Z","iopub.status.idle":"2022-07-19T13:24:13.326769Z","shell.execute_reply.started":"2022-07-19T13:24:13.283550Z","shell.execute_reply":"2022-07-19T13:24:13.325826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0\ta261b6e14276\t0.0102\t0.3887\t0.5911\n# 1\t5a88900e7dc1\t0.0309\t0.8405\t0.1185\n# 2\t9790d835736b\t0.0217\t0.6997\t0.2686\n# 3\t75ce6d68b67b\t0.0512\t0.6365\t0.3023\n# 4\t93578d946723\t0.0399\t0.6053\t0.3448\n# 5\t2e214524dbe3\t0.0099\t0.3721\t0.6080\n# 6\t84812fc2ab9f\t0.0084\t0.2796\t0.7020\n# 7\tc668ff840720\t0.0171\t0.5888\t0.3841\n# 8\t739a6d00f44a\t0.0178\t0.4030\t0.5692\n# 9\tbcfae2c9a244\t0.0132\t0.6373\t0.3395","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.021768,"end_time":"2022-06-29T02:56:11.116966","exception":false,"start_time":"2022-06-29T02:56:11.095198","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:24:14.559484Z","iopub.execute_input":"2022-07-19T13:24:14.560220Z","iopub.status.idle":"2022-07-19T13:24:14.563673Z","shell.execute_reply.started":"2022-07-19T13:24:14.560182Z","shell.execute_reply":"2022-07-19T13:24:14.562944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","metadata":{"papermill":{"duration":0.027941,"end_time":"2022-06-29T02:56:11.160924","exception":false,"start_time":"2022-06-29T02:56:11.132983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-19T13:24:17.842824Z","iopub.execute_input":"2022-07-19T13:24:17.843520Z","iopub.status.idle":"2022-07-19T13:24:17.852837Z","shell.execute_reply.started":"2022-07-19T13:24:17.843482Z","shell.execute_reply":"2022-07-19T13:24:17.851940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}