{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\nThe below inference code is my current submission code(2022-06-27). As I continuously learn from the community, I've gained much knowledge about model building and tuning. I want to share my current progress and will keep moving forward and welcome new insights from the community.  \nThis submission code combines the great work from below links. I really learned a lot from the masters.  \n1. [PPPM / Deberta-v3-large baseline w/ W&B [train]](https://www.kaggle.com/code/yasufuminakama/pppm-deberta-v3-large-baseline-w-w-b-train)  \n2. [Optimization approaches for Transformers](https://www.kaggle.com/code/vad13irt/optimization-approaches-for-transformers/notebook)  \n3. [4th place solution of 2021-feedback](https://www.kaggle.com/competitions/feedback-prize-2021/discussion/313330#1724876)  \n\n## So far useful attempts  \n* 512 max length\n* WeightedLayerPooling (Slightly improve deberta-base model from simple \\[CLS] head) \n* GroupFold \n* Different learning rates across layers\n* Preprocessing (encoding-resolve+normalize) \n\n## About to try\n* Currently doing more EDA, trying to find out more potential gaining point from the text data instead of just tuning hyper-parameters :)  \n* Fix my deberta-v3 models :(\n* Working on 1024 max length.\n\n## CV / LB  \n* deberta-base   CV:0.662 / LB: 0.631\n* deberta-large  CV:0.637 / LB: 0.619","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport shutil\n\nfrom torch.utils.data import DataLoader, Dataset\nimport datasets, transformers\nfrom transformers import TrainingArguments, Trainer\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer, DataCollatorWithPadding\n\nimport os\nimport gc\nimport re\nimport ast\nimport sys\nimport copy\nimport json\nimport time\nimport math\nimport shutil\nimport string\nimport pickle\nimport random\nimport joblib\nimport itertools\nfrom pathlib import Path\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom sklearn.metrics import f1_score, log_loss\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nimport torch\nprint(f\"torch.__version__: {torch.__version__}\")\nimport torch.nn as nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\n\n# os.system('pip uninstall -y transformers')\n# os.system('pip uninstall -y tokenizers')\n# os.system('python -m pip install --no-index --find-links=../input/pppm-pip-wheels transformers')\n# os.system('python -m pip install --no-index --find-links=../input/pppm-pip-wheels tokenizers')\nimport tokenizers\nimport transformers\nprint(f\"tokenizers.__version__: {tokenizers.__version__}\")\nprint(f\"transformers.__version__: {transformers.__version__}\")\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n%env TOKENIZERS_PARALLELISM=true\n\ngc.collect()\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-07-15T09:11:20.978807Z","iopub.execute_input":"2022-07-15T09:11:20.980019Z","iopub.status.idle":"2022-07-15T09:11:32.626694Z","shell.execute_reply.started":"2022-07-15T09:11:20.979887Z","shell.execute_reply":"2022-07-15T09:11:32.625466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    num_workers=1\n    path=\"../input/first-trial/\"\n    config_path=path+'config.pth'\n    #config_path='../input/feedback-deberta-large-051/config.pth'\n    model=\"microsoft/deberta-base\"\n    batch_size=16\n    fc_dropout=0.2\n    target_size=3\n    max_len=256\n    seed=42\n    n_fold=4\n    trn_fold=[0]\n    gradient_checkpoint=False","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:32.628694Z","iopub.execute_input":"2022-07-15T09:11:32.629561Z","iopub.status.idle":"2022-07-15T09:11:32.636090Z","shell.execute_reply.started":"2022-07-15T09:11:32.629477Z","shell.execute_reply":"2022-07-15T09:11:32.635048Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\n\ndef get_essay(essay_id, is_train=True):\n    parent_path = INPUT_DIR + 'train' if is_train else INPUT_DIR + 'test'\n    essay_path = os.path.join(parent_path, f\"{essay_id}.txt\")\n    essay_text = open(essay_path, 'r').read()\n    return essay_text\n\ndef softmax(z):\n    assert len(z.shape) == 2\n    s = np.max(z, axis=1)\n    s = s[:, np.newaxis] # necessary step to do broadcasting\n    e_x = np.exp(z - s)\n    div = np.sum(e_x, axis=1)\n    div = div[:, np.newaxis] # dito\n    return e_x / div\n\ndef get_score(y_true, y_pred):\n    y_pred = softmax(y_pred)\n    score = log_loss(y_true, y_pred)\n    return round(score, 5)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:32.637314Z","iopub.execute_input":"2022-07-15T09:11:32.638212Z","iopub.status.idle":"2022-07-15T09:11:32.656031Z","shell.execute_reply.started":"2022-07-15T09:11:32.638169Z","shell.execute_reply":"2022-07-15T09:11:32.654708Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = \"../input/feedback-prize-effectiveness/\"\n\n# train = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\n# submission = pd.read_csv(os.path.join(INPUT_DIR, 'sample_submission.csv'))\n# train['essay_text']  = train['essay_id'].apply(lambda x: get_essay(x, is_train=True))\n\ntest = pd.read_csv(os.path.join(INPUT_DIR, 'test.csv'))\nsubmission = pd.read_csv(os.path.join(INPUT_DIR, 'sample_submission.csv'))\ntest['essay_text']  = test['essay_id'].apply(lambda x: get_essay(x, is_train=False))\n","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:32.658279Z","iopub.execute_input":"2022-07-15T09:11:32.659142Z","iopub.status.idle":"2022-07-15T09:11:32.701447Z","shell.execute_reply.started":"2022-07-15T09:11:32.659085Z","shell.execute_reply":"2022-07-15T09:11:32.700576Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# tokenizer\n# ====================================================\ntokenizer = AutoTokenizer.from_pretrained(CFG.path + 'tokenizer')\nCFG.tokenizer = tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:32.703650Z","iopub.execute_input":"2022-07-15T09:11:32.704917Z","iopub.status.idle":"2022-07-15T09:11:32.935901Z","shell.execute_reply.started":"2022-07-15T09:11:32.704866Z","shell.execute_reply":"2022-07-15T09:11:32.934725Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\ndef replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n# Register the encoding and decoding error handlers for `utf-8` and `cp1252`.\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n    \"\"\"Resolve the encoding problems and normalize the abnormal characters.\"\"\"\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    text = unidecode(text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:32.937389Z","iopub.execute_input":"2022-07-15T09:11:32.937848Z","iopub.status.idle":"2022-07-15T09:11:32.960429Z","shell.execute_reply.started":"2022-07-15T09:11:32.937810Z","shell.execute_reply":"2022-07-15T09:11:32.959104Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# train['discourse_text'] = train['discourse_text'].apply(lambda x : resolve_encodings_and_normalize(x))\n# train['essay_text'] = train['essay_text'].apply(lambda x : resolve_encodings_and_normalize(x))\n\ntest['discourse_text'] = test['discourse_text'].apply(lambda x : resolve_encodings_and_normalize(x))\ntest['essay_text'] = test['essay_text'].apply(lambda x : resolve_encodings_and_normalize(x))","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:32.962416Z","iopub.execute_input":"2022-07-15T09:11:32.962993Z","iopub.status.idle":"2022-07-15T09:11:32.983019Z","shell.execute_reply.started":"2022-07-15T09:11:32.962943Z","shell.execute_reply":"2022-07-15T09:11:32.982004Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEP = tokenizer.sep_token\ntest['text'] = test['discourse_type'] + ' ' + test['discourse_text'] + SEP + test['essay_text']\ntest['label'] = np.nan\ndisplay(test.head())","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:32.984340Z","iopub.execute_input":"2022-07-15T09:11:32.984781Z","iopub.status.idle":"2022-07-15T09:11:33.013862Z","shell.execute_reply.started":"2022-07-15T09:11:32.984742Z","shell.execute_reply":"2022-07-15T09:11:33.013038Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.text = df['text'].values\n        self.label = df['label'].values\n\n    def __len__(self):\n        return len(self.text)\n\n    def __getitem__(self, item):\n        inputs = self.cfg.tokenizer.encode_plus(\n                        self.text[item],\n                        truncation=True,\n                        add_special_tokens=True,\n                        max_length=self.cfg.max_len\n                    )\n        samples = {\n            'input_ids': inputs['input_ids'],\n            'attention_mask': inputs['attention_mask'],\n        }\n\n        if 'token_type_ids' in inputs:\n            samples['token_type_ids'] = inputs['token_type_ids']\n        \n        return samples","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:33.015390Z","iopub.execute_input":"2022-07-15T09:11:33.016024Z","iopub.status.idle":"2022-07-15T09:11:33.024607Z","shell.execute_reply.started":"2022-07-15T09:11:33.015986Z","shell.execute_reply":"2022-07-15T09:11:33.023405Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Collate:\n    def __init__(self, tokenizer, isTrain=True):\n        self.tokenizer = tokenizer\n        self.isTrain = isTrain\n        # self.args = args\n\n    def __call__(self, batch):\n        output = dict()\n        output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n        output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n        if self.isTrain:\n            output[\"target\"] = [sample[\"target\"] for sample in batch]\n\n        # calculate max token length of this batch\n        batch_max = max([len(ids) for ids in output[\"input_ids\"]])\n\n        # add padding\n        if self.tokenizer.padding_side == \"right\":\n            output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n        else:\n            output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n\n        # convert to tensors\n        output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n        output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n        if self.isTrain:\n            output[\"target\"] = torch.tensor(output[\"target\"], dtype=torch.long)\n\n        return output\n\ncollate_fn = Collate(CFG.tokenizer, isTrain=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:33.027763Z","iopub.execute_input":"2022-07-15T09:11:33.028392Z","iopub.status.idle":"2022-07-15T09:11:33.044174Z","shell.execute_reply.started":"2022-07-15T09:11:33.028349Z","shell.execute_reply":"2022-07-15T09:11:33.043229Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MeanPooling(nn.Module):\n    def __init__(self):\n        super(MeanPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n        sum_mask = input_mask_expanded.sum(1)\n        sum_mask = torch.clamp(sum_mask, min=1e-9)\n        mean_embeddings = sum_embeddings / sum_mask\n        return mean_embeddings\n\n    \nclass MeanMaxPooling(nn.Module):\n    def __init__(self):\n        super(MeanMaxPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        mean_pooling_embeddings = torch.mean(last_hidden_state, 1)\n        _, max_pooling_embeddings = torch.max(last_hidden_state, 1)\n        mean_max_embeddings = torch.cat((mean_pooling_embeddings, max_pooling_embeddings), 1)\n        return mean_max_embeddings\n\n    \nclass LSTMPooling(nn.Module):\n    def __init__(self, num_layers, hidden_size, hiddendim_lstm):\n        super(LSTMPooling, self).__init__()\n        self.num_hidden_layers = num_layers\n        self.hidden_size = hidden_size\n        self.hiddendim_lstm = hiddendim_lstm\n        self.lstm = nn.LSTM(self.hidden_size, self.hiddendim_lstm, batch_first=True)\n        self.dropout = nn.Dropout(0.1)\n    \n    def forward(self, all_hidden_states):\n        ## forward\n        hidden_states = torch.stack([all_hidden_states[layer_i][:, 0].squeeze()\n                                     for layer_i in range(1, self.num_hidden_layers+1)], dim=-1)\n        hidden_states = hidden_states.view(-1, self.num_hidden_layers, self.hidden_size)\n        out, _ = self.lstm(hidden_states, None)\n        out = self.dropout(out[:, -1, :])\n        return out\n    \nclass WeightedLayerPooling(nn.Module):\n    def __init__(self, num_hidden_layers, layer_start: int = 4, layer_weights = None):\n        super(WeightedLayerPooling, self).__init__()\n        self.layer_start = layer_start\n        self.num_hidden_layers = num_hidden_layers\n        self.layer_weights = layer_weights if layer_weights is not None \\\n            else nn.Parameter(\n                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float)\n            )\n\n    def forward(self, all_hidden_states):\n        all_layer_embedding = all_hidden_states[self.layer_start:, :, :, :]\n        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n        return weighted_average","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:33.045663Z","iopub.execute_input":"2022-07-15T09:11:33.046923Z","iopub.status.idle":"2022-07-15T09:11:33.068379Z","shell.execute_reply.started":"2022-07-15T09:11:33.046875Z","shell.execute_reply":"2022-07-15T09:11:33.067177Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Model\n# ====================================================\nfrom torch.cuda.amp import autocast\nclass CustomModel(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg\n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n        else:\n            self.model = AutoModel.from_config(self.config)\n        \n        # gradient checkpointing\n        if self.cfg.gradient_checkpoint:\n            self.model.gradient_checkpointing_enable()\n            print(f\"Gradient Checkpointing: {self.model.is_gradient_checkpointing}\")\n            \n        \n        # self.pooler = MeanPooling()\n        \n        self.bilstm = nn.LSTM(self.config.hidden_size, (self.config.hidden_size) // 2, num_layers=2, \n                              dropout=self.config.hidden_dropout_prob, batch_first=True,\n                              bidirectional=True)\n        \n        self.dropout = nn.Dropout(0.2)\n        self.dropout1 = nn.Dropout(0.1)\n        self.dropout2 = nn.Dropout(0.2)\n        self.dropout3 = nn.Dropout(0.3)\n        self.dropout4 = nn.Dropout(0.4)\n        self.dropout5 = nn.Dropout(0.5)\n        \n        self.output = nn.Sequential(\n            nn.Linear(self.config.hidden_size, self.cfg.target_size)\n            # nn.Linear(256, self.cfg.target_size)\n        )\n        \n        \n\n    def loss(self, outputs, targets):\n        loss_fct = nn.CrossEntropyLoss()\n        loss = loss_fct(outputs, targets)\n        return loss\n    \n    def monitor_metrics(self, outputs, targets):\n        device = targets.get_device()\n        # print(outputs)\n        # print(targets)\n        mll = log_loss(\n            targets.cpu().detach().numpy(),\n            softmax(outputs.cpu().detach().numpy()),\n            labels=[0, 1, 2],\n        )\n        return mll\n        \n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.Embedding):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n\n    def forward(self, ids, mask, token_type_ids=None, targets=None):\n        if token_type_ids:\n            transformer_out = self.model(ids, mask, token_type_ids)\n        else:\n            transformer_out = self.model(ids, mask)\n        \n        # LSTM/GRU header\n#         all_hidden_states = torch.stack(transformer_out[1])\n#         sequence_output = self.pooler(all_hidden_states)\n        \n        # simple CLS\n        sequence_output = transformer_out[0][:, 0, :]\n\n        \n        # Main task\n        logits1 = self.output(self.dropout1(sequence_output))\n        logits2 = self.output(self.dropout2(sequence_output))\n        logits3 = self.output(self.dropout3(sequence_output))\n        logits4 = self.output(self.dropout4(sequence_output))\n        logits5 = self.output(self.dropout5(sequence_output))\n        logits = (logits1 + logits2 + logits3 + logits4 + logits5) / 5\n\n        if targets is not None:\n            metric = self.monitor_metrics(logits, targets)\n            return logits, metric\n        \n        return logits, 0.","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:33.070765Z","iopub.execute_input":"2022-07-15T09:11:33.071299Z","iopub.status.idle":"2022-07-15T09:11:33.098082Z","shell.execute_reply.started":"2022-07-15T09:11:33.071239Z","shell.execute_reply":"2022-07-15T09:11:33.096831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# inference\n# ====================================================\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    for data in tk0:\n        ids = data['input_ids'].to(device, dtype = torch.long)\n        mask = data['attention_mask'].to(device, dtype = torch.long)\n        with torch.no_grad():\n            y_preds, _ = model(ids, mask)\n        y_preds = softmax(y_preds.to('cpu').numpy())\n        preds.append(y_preds)\n    predictions = np.concatenate(preds)\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:33.099581Z","iopub.execute_input":"2022-07-15T09:11:33.100473Z","iopub.status.idle":"2022-07-15T09:11:33.115352Z","shell.execute_reply.started":"2022-07-15T09:11:33.100430Z","shell.execute_reply":"2022-07-15T09:11:33.114195Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deberta_predictions = []\ntest_dataset = TestDataset(CFG, test)\ntest_loader = DataLoader(test_dataset,\n                         batch_size=CFG.batch_size,\n                         shuffle=False,\n                         collate_fn=collate_fn,\n                         num_workers=CFG.num_workers, pin_memory=True, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:33.116879Z","iopub.execute_input":"2022-07-15T09:11:33.117396Z","iopub.status.idle":"2022-07-15T09:11:33.133119Z","shell.execute_reply.started":"2022-07-15T09:11:33.117355Z","shell.execute_reply":"2022-07-15T09:11:33.131668Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deberta_predictions = []\n#for fold in CFG.trn_fold:\nfor fold in [0]:\n    print(\"Fold {}\".format(fold))\n\n    model = CustomModel(CFG, config_path=CFG.config_path, pretrained=False)\n    state = torch.load(CFG.path+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\",\n                       map_location=torch.device('cpu'))\n    model.load_state_dict(state['model'])\n    prediction = inference_fn(test_loader, model, device)\n    deberta_predictions.append(prediction)\n    del model, state, prediction; gc.collect()\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:33.135010Z","iopub.execute_input":"2022-07-15T09:11:33.135856Z","iopub.status.idle":"2022-07-15T09:11:59.377697Z","shell.execute_reply.started":"2022-07-15T09:11:33.135796Z","shell.execute_reply":"2022-07-15T09:11:59.376018Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predictions = np.mean(deberta_predictions, axis=0)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:59.380849Z","iopub.execute_input":"2022-07-15T09:11:59.381648Z","iopub.status.idle":"2022-07-15T09:11:59.389277Z","shell.execute_reply.started":"2022-07-15T09:11:59.381579Z","shell.execute_reply":"2022-07-15T09:11:59.387739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = CustomModel(CFG, config_path=CFG.config_path, pretrained=False)\nstate = torch.load(CFG.path+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\",\n                   map_location=torch.device('cpu'))\nmodel.load_state_dict(state['model'])","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:11:59.391012Z","iopub.execute_input":"2022-07-15T09:11:59.391730Z","iopub.status.idle":"2022-07-15T09:12:02.395622Z","shell.execute_reply.started":"2022-07-15T09:11:59.391685Z","shell.execute_reply":"2022-07-15T09:12:02.394477Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['Ineffective'] = predictions[:, 2]\nsubmission['Adequate'] = predictions[:, 0]\nsubmission['Effective'] = predictions[:, 1]\n\ndisplay(submission)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:13:50.074278Z","iopub.execute_input":"2022-07-15T09:13:50.074806Z","iopub.status.idle":"2022-07-15T09:13:50.093421Z","shell.execute_reply.started":"2022-07-15T09:13:50.074768Z","shell.execute_reply":"2022-07-15T09:13:50.092554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:12:02.418308Z","iopub.execute_input":"2022-07-15T09:12:02.419043Z","iopub.status.idle":"2022-07-15T09:12:02.430026Z","shell.execute_reply.started":"2022-07-15T09:12:02.418990Z","shell.execute_reply":"2022-07-15T09:12:02.428429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"CFG.tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-15T09:12:02.432255Z","iopub.execute_input":"2022-07-15T09:12:02.432962Z","iopub.status.idle":"2022-07-15T09:12:02.443965Z","shell.execute_reply.started":"2022-07-15T09:12:02.432903Z","shell.execute_reply":"2022-07-15T09:12:02.442902Z"},"trusted":true},"execution_count":null,"outputs":[]}]}