{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport gc\nimport math\nimport time\nimport random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\nimport warnings\nwarnings.simplefilter('ignore')\nfrom tqdm import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nimport torch.optim as optim\nfrom torch.optim import lr_scheduler\nfrom torch.utils.data import DataLoader, Dataset\n\nfrom sklearn.model_selection import StratifiedKFold\nfrom sklearn.metrics import log_loss\n\nfrom transformers import AutoModel, AutoConfig, AutoTokenizer, AdamW, DataCollatorWithPadding\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\nfrom torch.cuda.amp import autocast, GradScaler\n# https://github.com/huggingface/transformers/issues/9919\nfrom torch.utils.checkpoint import checkpoint\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\nimport pickle # For roberta test","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:26.152767Z","iopub.execute_input":"2022-07-12T05:06:26.153365Z","iopub.status.idle":"2022-07-12T05:06:34.691555Z","shell.execute_reply.started":"2022-07-12T05:06:26.153245Z","shell.execute_reply":"2022-07-12T05:06:34.690494Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = '../input/feedback-prize-effectiveness/'","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:34.693856Z","iopub.execute_input":"2022-07-12T05:06:34.694628Z","iopub.status.idle":"2022-07-12T05:06:34.701330Z","shell.execute_reply.started":"2022-07-12T05:06:34.694585Z","shell.execute_reply":"2022-07-12T05:06:34.699201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<h1 style = \"font-size:60px; font-family:Garamond ; font-weight : normal; background-color: #f6f5f5 ; color : #fe346e; text-align: center; border-radius: 100px 100px;\">Deberta V3 Base</h1>\n<br>","metadata":{}},{"cell_type":"markdown","source":"# CFG","metadata":{}},{"cell_type":"code","source":"class CFG:\n    wandb = False\n    apex = True #\n    model = '../input/deberta-v3-base/deberta-v3-base'\n    fast = True\n    seed = 42\n    n_splits = 5\n    max_len = 512\n    dropout = 0.1\n    target_size = 3\n    print_freq = 50\n    min_lr = 1e-6\n    scheduler = 'cosine'\n    batch_size = 8\n    num_workers = 0\n    lr = 3e-5\n    weigth_decay = 0.01\n    epochs = 3\n    n_fold = 5\n    trn_fold = [0, 1, 2, 3, 4]\n    train = True \n    num_warmup_steps = 0 #\n    num_cycles=0.5 #\n    CVs = []\n    debug = False\n    debug_ver2 = False\n    gradient_checkpointing = True\n    AMP = False\n    freezing = True\n    # after_freezed_parameters = []\n    \n    n_accumulate= 1\n    \n\nif CFG.debug:\n    CFG.epochs = 2\n    CFG.trn_fold = [0, 1]\n    CFG.print_freq = 10\n\nif CFG.debug_ver2:\n    CFG.epochs = 1\n    CFG.trn_fold = [0, 1]","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:34.703339Z","iopub.execute_input":"2022-07-12T05:06:34.703748Z","iopub.status.idle":"2022-07-12T05:06:34.715152Z","shell.execute_reply.started":"2022-07-12T05:06:34.703709Z","shell.execute_reply":"2022-07-12T05:06:34.713934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helper Function","metadata":{}},{"cell_type":"code","source":"# Loss Func\ndef criterion(outputs, labels):\n    return nn.CrossEntropyLoss()(outputs, labels)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:34.718793Z","iopub.execute_input":"2022-07-12T05:06:34.719268Z","iopub.status.idle":"2022-07-12T05:06:34.726590Z","shell.execute_reply.started":"2022-07-12T05:06:34.719223Z","shell.execute_reply":"2022-07-12T05:06:34.725568Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def softmax(z):\n    assert len(z.shape) == 2\n    s = np.max(z, axis=1)\n    s = s[:, np.newaxis] # necessary step to do broadcasting\n    e_x = np.exp(z - s)\n    div = np.sum(e_x, axis=1)\n    div = div[:, np.newaxis] # dito\n    return e_x / div","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:34.728578Z","iopub.execute_input":"2022-07-12T05:06:34.729031Z","iopub.status.idle":"2022-07-12T05:06:34.737799Z","shell.execute_reply.started":"2022-07-12T05:06:34.728987Z","shell.execute_reply":"2022-07-12T05:06:34.736442Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def freeze(module):\n    \"\"\"\n    Freezes module's parameters.\n    \"\"\"\n    \n    for parameter in module.parameters():\n        parameter.requires_grad = False\n        \ndef get_freezed_parameters(module):\n    \"\"\"\n    Returns names of freezed parameters of the given module.\n    \"\"\"\n    \n    freezed_parameters = []\n    for name, parameter in module.named_parameters():\n        if not parameter.requires_grad:\n            freezed_parameters.append(name)\n            \n    return freezed_parameters","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:34.740168Z","iopub.execute_input":"2022-07-12T05:06:34.740812Z","iopub.status.idle":"2022-07-12T05:06:34.751340Z","shell.execute_reply.started":"2022-07-12T05:06:34.740766Z","shell.execute_reply":"2022-07-12T05:06:34.750110Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# append to train/test.csv\ndef get_essay(essay_id, is_train=True):\n    parent_path = INPUT_DIR + 'train' if is_train else INPUT_DIR + 'test' \n    essay_path = os.path.join(parent_path, f\"{essay_id}.txt\")\n    essay_text = open(essay_path, 'r').read()\n    return essay_text","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:34.753043Z","iopub.execute_input":"2022-07-12T05:06:34.753976Z","iopub.status.idle":"2022-07-12T05:06:34.762530Z","shell.execute_reply.started":"2022-07-12T05:06:34.753942Z","shell.execute_reply":"2022-07-12T05:06:34.761424Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Preprocess & Tokenizer","metadata":{}},{"cell_type":"code","source":"# Testing Data\ntest = pd.read_csv(INPUT_DIR + 'test.csv')\ntest['essay_text'] = test['essay_id'].apply(lambda x: get_essay(x, is_train=False))\ntest.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:34.764289Z","iopub.execute_input":"2022-07-12T05:06:34.764897Z","iopub.status.idle":"2022-07-12T05:06:34.813482Z","shell.execute_reply.started":"2022-07-12T05:06:34.764792Z","shell.execute_reply":"2022-07-12T05:06:34.812358Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.fast:\n    tokenizer = AutoTokenizer.from_pretrained(CFG.model, use_fast=True)\nelse:\n    tokenizer = AutoTokenizer.from_pretrained(CFG.model)\nCFG.tokenizer = tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:34.816041Z","iopub.execute_input":"2022-07-12T05:06:34.816813Z","iopub.status.idle":"2022-07-12T05:06:35.625413Z","shell.execute_reply.started":"2022-07-12T05:06:34.816772Z","shell.execute_reply":"2022-07-12T05:06:35.624400Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Normalize","metadata":{}},{"cell_type":"code","source":"from text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\ndef replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n# Register the encoding and decoding error handlers for `utf-8` and `cp1252`.\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n    \"\"\"Resolve the encoding problems and normalize the abnormal characters.\"\"\"\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    text = unidecode(text)\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:35.630467Z","iopub.execute_input":"2022-07-12T05:06:35.631346Z","iopub.status.idle":"2022-07-12T05:06:35.653306Z","shell.execute_reply.started":"2022-07-12T05:06:35.631298Z","shell.execute_reply":"2022-07-12T05:06:35.652220Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['discourse_text'] = test['discourse_text'].apply(lambda x : resolve_encodings_and_normalize(x))\ntest['essay_text'] = test['essay_text'].apply(lambda x : resolve_encodings_and_normalize(x))\n# Tokenize the test data\ntest['text'] = test['discourse_type'] + ' '+ test['discourse_text'] + '[SEP]' + test['essay_text']\ntest['label'] = np.nan","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:35.655649Z","iopub.execute_input":"2022-07-12T05:06:35.656104Z","iopub.status.idle":"2022-07-12T05:06:35.676766Z","shell.execute_reply.started":"2022-07-12T05:06:35.656065Z","shell.execute_reply":"2022-07-12T05:06:35.675831Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{}},{"cell_type":"code","source":"# Testing Datasets\nclass TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.text = df['text'].values\n\n    def __len__(self):\n        return len(self.text)\n\n    def __getitem__(self, item):\n        inputs = self.cfg.tokenizer.encode_plus(\n                        self.text[item],\n                        truncation=True,\n                        add_special_tokens=True,\n                        max_length=self.cfg.max_len\n                    )\n        samples = {\n            'input_ids': inputs['input_ids'],\n            'attention_mask': inputs['attention_mask'],\n        }\n\n        if 'token_type_ids' in inputs:\n            samples['token_type_ids'] = inputs['token_type_ids']\n        \n        return samples","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:35.678391Z","iopub.execute_input":"2022-07-12T05:06:35.678733Z","iopub.status.idle":"2022-07-12T05:06:35.689433Z","shell.execute_reply.started":"2022-07-12T05:06:35.678697Z","shell.execute_reply":"2022-07-12T05:06:35.688495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dynamic padding","metadata":{}},{"cell_type":"code","source":"# Dynamic Padding (Collate)\n# collate_fn = DataCollatorWithPadding(tokenizer=CFG.tokenizer)\nclass Collate:\n    def __init__(self, tokenizer, isTrain=True):\n        self.tokenizer = tokenizer\n        self.isTrain = isTrain\n        # self.args = args\n\n    def __call__(self, batch):\n        output = dict()\n        output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n        output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n        if self.isTrain:\n            output[\"target\"] = [sample[\"target\"] for sample in batch]\n\n        # calculate max token length of this batch\n        batch_max = max([len(ids) for ids in output[\"input_ids\"]])\n\n        # add padding\n        if self.tokenizer.padding_side == \"right\":\n            output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n        else:\n            output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n            output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n\n        # convert to tensors\n        output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n        output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n        if self.isTrain:\n            output[\"target\"] = torch.tensor(output[\"target\"], dtype=torch.long)\n\n        return output","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:35.691621Z","iopub.execute_input":"2022-07-12T05:06:35.692252Z","iopub.status.idle":"2022-07-12T05:06:35.707408Z","shell.execute_reply.started":"2022-07-12T05:06:35.692207Z","shell.execute_reply":"2022-07-12T05:06:35.706255Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{}},{"cell_type":"code","source":"class MeanPooling(nn.Module):\n    def __init__(self):\n        super(MeanPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n        sum_mask = input_mask_expanded.sum(1)\n        sum_mask = torch.clamp(sum_mask, min=1e-9) #\n        mean_embeddings = sum_embeddings / sum_mask\n        return mean_embeddings","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:35.708717Z","iopub.execute_input":"2022-07-12T05:06:35.709023Z","iopub.status.idle":"2022-07-12T05:06:35.722057Z","shell.execute_reply.started":"2022-07-12T05:06:35.708997Z","shell.execute_reply":"2022-07-12T05:06:35.721011Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedBackModel(nn.Module):\n    def __init__(self, model_name):\n        super(FeedBackModel, self).__init__()\n        # Header (fast or normal)\n        self.model = AutoModel.from_pretrained(model_name)\n        \n        # Gradient_checkpointing\n        if CFG.gradient_checkpointing:\n            (self.model).gradient_checkpointing_enable()\n        \n        # Freezing\n        if CFG.freezing:\n            # freezing embeddings and first 2 layers of encoder\n            freeze((self.model).embeddings)\n            freeze((self.model).encoder.layer[:2])\n            CFG.after_freezed_parameters = filter(lambda parameter: parameter.requires_grad, (self.model).parameters())\n        \n        self.config = AutoConfig.from_pretrained(model_name)\n        self.drop = nn.Dropout(p=CFG.dropout)\n        self.pooler = MeanPooling()\n        self.fc = nn.Linear(self.config.hidden_size, CFG.target_size)\n        \n    def forward(self, ids, mask):        \n        out = self.model(input_ids=ids, \n                         attention_mask=mask,\n                         output_hidden_states=False)\n        out = self.pooler(out.last_hidden_state, mask)\n        out = self.drop(out)\n        outputs = self.fc(out)\n        return outputs","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:35.723885Z","iopub.execute_input":"2022-07-12T05:06:35.724322Z","iopub.status.idle":"2022-07-12T05:06:35.737410Z","shell.execute_reply.started":"2022-07-12T05:06:35.724278Z","shell.execute_reply":"2022-07-12T05:06:35.736292Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Prediction","metadata":{}},{"cell_type":"code","source":"# predict the test value result\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    for data in tk0:\n        ids = data['input_ids'].to(device, dtype = torch.long)\n        mask = data['attention_mask'].to(device, dtype = torch.long)\n        with torch.no_grad():\n            y_preds = model(ids, mask)\n        y_preds = softmax(y_preds.to('cpu').numpy())\n        # y_preds = y_preds.to('cpu').numpy()\n        preds.append(y_preds)\n    predictions = np.concatenate(preds)\n    return predictions","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:35.738656Z","iopub.execute_input":"2022-07-12T05:06:35.738966Z","iopub.status.idle":"2022-07-12T05:06:35.749934Z","shell.execute_reply.started":"2022-07-12T05:06:35.738939Z","shell.execute_reply":"2022-07-12T05:06:35.748936Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testDataset = TestDataset(CFG, test)\ntest_loader = DataLoader(testDataset,\n                              batch_size = CFG.batch_size,\n                              shuffle=False,\n                              collate_fn = Collate(CFG.tokenizer, isTrain=False),\n                              num_workers = CFG.num_workers,\n                              pin_memory = True,\n                              drop_last=False)\ndeberta_predictions = []\nfor i in CFG.trn_fold:\n    model = FeedBackModel(CFG.model)\n    model.load_state_dict(torch.load('../input/dbv3basemodels202279/models-deberta-v3-base-deberta-v3-base_fold' + str(i) +'_best.pth'))\n    prediction = inference_fn(test_loader, model, device)\n    deberta_predictions.append(prediction)\n    torch.cuda.empty_cache()\n    gc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:06:35.751524Z","iopub.execute_input":"2022-07-12T05:06:35.752117Z","iopub.status.idle":"2022-07-12T05:07:40.312381Z","shell.execute_reply.started":"2022-07-12T05:06:35.752078Z","shell.execute_reply":"2022-07-12T05:07:40.311334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Save the predictions","metadata":{}},{"cell_type":"code","source":"deb_ineffective = []\ndeb_effective = []\ndeb_adequate = []\n\nfor x in deberta_predictions:\n    deb_ineffective.append(x[:, 0])\n    deb_adequate.append(x[:, 1])\n    deb_effective.append(x[:, 2])\n# list -> dataframe\ndeb_ineffective = pd.DataFrame(deb_ineffective).T\ndeb_adequate = pd.DataFrame(deb_adequate).T\ndeb_effective = pd.DataFrame(deb_effective).T","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.314680Z","iopub.execute_input":"2022-07-12T05:07:40.315439Z","iopub.status.idle":"2022-07-12T05:07:40.326222Z","shell.execute_reply.started":"2022-07-12T05:07:40.315383Z","shell.execute_reply":"2022-07-12T05:07:40.325119Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<h1 style = \"font-size:60px; font-family:Garamond ; font-weight : normal; background-color: #f6f5f5 ; color : #fe346e; text-align: center; border-radius: 100px 100px;\">Roberta Base + Deberta-Large</h1>\n<br>","metadata":{}},{"cell_type":"markdown","source":"reference:https://www.kaggle.com/code/renokan/fork-ensemble-deberta-roberta/notebook","metadata":{}},{"cell_type":"markdown","source":"# Pre process","metadata":{}},{"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport glob\n\nfrom text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\n\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.nn import Parameter\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom transformers import AutoModel, AutoTokenizer, AutoConfig\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.327698Z","iopub.execute_input":"2022-07-12T05:07:40.328072Z","iopub.status.idle":"2022-07-12T05:07:40.340109Z","shell.execute_reply.started":"2022-07-12T05:07:40.328034Z","shell.execute_reply":"2022-07-12T05:07:40.338970Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    \n    text = unidecode(text)\n    \n    return text\n\n\ndef fetch_essay(essay_id: str, txt_dir: str):\n    essay_path = os.path.join(COMP_DIR + txt_dir, essay_id + '.txt')\n    essay_text = open(essay_path, 'r').read()\n    \n    return essay_text\n\n\ndef prepare_input(cfg, text, text_2=None):\n    inputs = cfg.tokenizer(text, text_2,\n                           padding=\"max_length\",\n                           add_special_tokens=True,\n                           max_length=cfg.max_len,\n                           truncation=True)\n\n    for k, v in inputs.items():\n        inputs[k] = torch.tensor(v, dtype=torch.long)\n        \n    return inputs\n\n\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    \n    for inputs in tk0:\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n            \n        with torch.no_grad():\n            output = model(inputs)\n        \n        preds.append(F.softmax(output).to('cpu').numpy())\n\n    return np.concatenate(preds)  \n\n\ndef show_gradient(df, n_row=None):\n    if not n_row:\n        n_row = 5\n\n    return df.head(n_row) \\\n                .assign(all_mean=lambda x: x.mean(axis=1)) \\\n                    .style.background_gradient(cmap=cm, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.342395Z","iopub.execute_input":"2022-07-12T05:07:40.343089Z","iopub.status.idle":"2022-07-12T05:07:40.360234Z","shell.execute_reply.started":"2022-07-12T05:07:40.343048Z","shell.execute_reply":"2022-07-12T05:07:40.358920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.precision', 4)\ncm = sns.light_palette('green', as_cmap=True)\nprops_param = \"color:white; font-weight:bold; background-color:green;\"\n\nN_ROW = 10\n\nCOMP_DIR = \"../input/feedback-prize-effectiveness/\"\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.362075Z","iopub.execute_input":"2022-07-12T05:07:40.362442Z","iopub.status.idle":"2022-07-12T05:07:40.377406Z","shell.execute_reply.started":"2022-07-12T05:07:40.362405Z","shell.execute_reply":"2022-07-12T05:07:40.376420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = COMP_DIR + \"test.csv\"\nsubmission_path = COMP_DIR + \"sample_submission.csv\"\n\ntest_origin = pd.read_csv(test_path)\nsubmission_origin = pd.read_csv(submission_path)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.379233Z","iopub.execute_input":"2022-07-12T05:07:40.379819Z","iopub.status.idle":"2022-07-12T05:07:40.398083Z","shell.execute_reply.started":"2022-07-12T05:07:40.379775Z","shell.execute_reply":"2022-07-12T05:07:40.397146Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_origin.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.399520Z","iopub.execute_input":"2022-07-12T05:07:40.399876Z","iopub.status.idle":"2022-07-12T05:07:40.413119Z","shell.execute_reply.started":"2022-07-12T05:07:40.399838Z","shell.execute_reply":"2022-07-12T05:07:40.411863Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_path = \"../input/feedback-prize-effectiveness/train.csv\"\ncols_list = ['essay_id', 'discourse_text']\nidxs_list = [49, 80, 945, 947, 1870]\n\ntemp = pd.read_csv(data_path, usecols=cols_list).loc[idxs_list, :]\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.415250Z","iopub.execute_input":"2022-07-12T05:07:40.415957Z","iopub.status.idle":"2022-07-12T05:07:40.692572Z","shell.execute_reply.started":"2022-07-12T05:07:40.415909Z","shell.execute_reply":"2022-07-12T05:07:40.691596Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp['discourse_text_UPD'] = temp['discourse_text'].apply(resolve_encodings_and_normalize)\n\ntemp['essay_text'] = temp['essay_id'].transform(fetch_essay, txt_dir='train')\ntemp['essay_text_UPD'] = temp['essay_text'].apply(resolve_encodings_and_normalize)\n\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.694570Z","iopub.execute_input":"2022-07-12T05:07:40.695221Z","iopub.status.idle":"2022-07-12T05:07:40.736439Z","shell.execute_reply.started":"2022-07-12T05:07:40.695182Z","shell.execute_reply":"2022-07-12T05:07:40.735421Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n, row in enumerate(temp.iterrows()):\n    indx, data = row\n    disc_text = data.discourse_text\n    disc_text_upd = data.discourse_text_UPD\n\n    print(f'\\nN{n} === index: {indx} ===')\n    print(f'\\n>>> origin text:')\n    print(repr(disc_text))\n    print(f'\\n>>> updated text:')\n    print(repr(disc_text_upd))","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.737903Z","iopub.execute_input":"2022-07-12T05:07:40.738236Z","iopub.status.idle":"2022-07-12T05:07:40.748423Z","shell.execute_reply.started":"2022-07-12T05:07:40.738199Z","shell.execute_reply":"2022-07-12T05:07:40.746361Z"},"jupyter":{"source_hidden":true},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Deberta Large","metadata":{}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.text = df['text'].values\n\n    def __len__(self):\n        return len(self.text)\n\n    def __getitem__(self, item):    \n        text = self.text[item]\n        inputs = prepare_input(self.cfg, text)\n        \n        return inputs\n\nclass CustomModel(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg\n        \n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n        else:\n            self.model = AutoModel.from_config(self.config)\n        \n        self.bilstm = nn.LSTM(self.config.hidden_size, (self.config.hidden_size) // 2, num_layers=2, \n                              dropout=self.config.hidden_dropout_prob, batch_first=True,\n                              bidirectional=True)\n        \n        # self.dropout = nn.Dropout(0.2)\n        self.dropout1 = nn.Dropout(0.1)\n        self.dropout2 = nn.Dropout(0.2)\n        self.dropout3 = nn.Dropout(0.3)\n        self.dropout4 = nn.Dropout(0.4)\n        self.dropout5 = nn.Dropout(0.5)\n        \n        self.output = nn.Sequential(\n            nn.Linear(self.config.hidden_size, 3)  # self.cfg.target_size\n        )\n                \n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.Embedding):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n\n    def forward(self, inputs):\n        sequence_output = self.model(**inputs)[0][:, 0, :]\n\n        logits1 = self.output(self.dropout1(sequence_output))\n        logits2 = self.output(self.dropout2(sequence_output))\n        logits3 = self.output(self.dropout3(sequence_output))\n        logits4 = self.output(self.dropout4(sequence_output))\n        logits5 = self.output(self.dropout5(sequence_output))\n        logits = (logits1 + logits2 + logits3 + logits4 + logits5) / 5\n\n        return logits","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.750418Z","iopub.execute_input":"2022-07-12T05:07:40.750830Z","iopub.status.idle":"2022-07-12T05:07:40.769835Z","shell.execute_reply.started":"2022-07-12T05:07:40.750791Z","shell.execute_reply":"2022-07-12T05:07:40.768697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    path = \"../input/feedback-deberta-large-051/\"\n    config_path = path+'config.pth'\n    model = \"microsoft/deberta-large\"\n    num_workers = 2\n    batch_size = 32\n    max_len = 512\n    seed = 42\n    n_fold = 4\n    # trn_fold = [0, 1, 2, 3]\n    # fc_dropout = 0.2\n    # target_size = 3\n    \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.path + 'tokenizer')","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.777358Z","iopub.execute_input":"2022-07-12T05:07:40.778147Z","iopub.status.idle":"2022-07-12T05:07:40.928426Z","shell.execute_reply.started":"2022-07-12T05:07:40.778101Z","shell.execute_reply":"2022-07-12T05:07:40.927325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test_origin.copy()\nSEP = CFG.tokenizer.sep_token\n\ndf['discourse_text'] = df['discourse_text'].apply(resolve_encodings_and_normalize)\ndf['essay_text'] = df['essay_id'].transform(fetch_essay, txt_dir='test')\ndf['essay_text'] = df['essay_text'].apply(resolve_encodings_and_normalize)\ndf['text'] = df['discourse_type'] + ' ' + df['discourse_text'] + SEP + df['essay_text']\n\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.929951Z","iopub.execute_input":"2022-07-12T05:07:40.930327Z","iopub.status.idle":"2022-07-12T05:07:40.961034Z","shell.execute_reply.started":"2022-07-12T05:07:40.930289Z","shell.execute_reply":"2022-07-12T05:07:40.959940Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(CFG, df)\ntest_loader = DataLoader(test_dataset,\n                         batch_size=CFG.batch_size,\n                         shuffle=False,\n                         num_workers=CFG.num_workers,\n                         pin_memory=True, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.962766Z","iopub.execute_input":"2022-07-12T05:07:40.963157Z","iopub.status.idle":"2022-07-12T05:07:40.968651Z","shell.execute_reply.started":"2022-07-12T05:07:40.963121Z","shell.execute_reply":"2022-07-12T05:07:40.967629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deberta_large_predictions = []\n\nfor fold in range(CFG.n_fold):\n    model = CustomModel(CFG, config_path=CFG.config_path, pretrained=False)\n    state = torch.load(CFG.path+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\",\n                       map_location=torch.device('cpu'))\n    \n    model.load_state_dict(state['model'])\n    prediction = inference_fn(test_loader, model, DEVICE)\n    \n    deberta_large_predictions.append(prediction)\n    \n    del model, state, prediction; gc.collect()\n    torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:07:40.970608Z","iopub.execute_input":"2022-07-12T05:07:40.970967Z","iopub.status.idle":"2022-07-12T05:09:27.370375Z","shell.execute_reply.started":"2022-07-12T05:07:40.970932Z","shell.execute_reply":"2022-07-12T05:09:27.369286Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_large_ineffective = []\ndeb_large_effective = []\ndeb_large_adequate = []\n\nfor x in deberta_large_predictions:\n    deb_large_ineffective.append(x[:, 0])\n    deb_large_adequate.append(x[:, 1])\n    deb_large_effective.append(x[:, 2])\n# list -> dataframe\ndeb_large_ineffective = pd.DataFrame(deb_large_ineffective).T\ndeb_large_adequate = pd.DataFrame(deb_large_adequate).T\ndeb_large_effective = pd.DataFrame(deb_large_effective).T","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:09:27.372308Z","iopub.execute_input":"2022-07-12T05:09:27.372978Z","iopub.status.idle":"2022-07-12T05:09:27.385278Z","shell.execute_reply.started":"2022-07-12T05:09:27.372937Z","shell.execute_reply":"2022-07-12T05:09:27.384243Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Roberta","metadata":{}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.discourse = df['discourse'].values\n        self.essay = df['essay'].values\n        \n    def __len__(self):\n        return len(self.discourse)\n    \n    def __getitem__(self, item):\n        discourse = self.discourse[item]\n        essay = self.essay[item]\n        \n        inputs = prepare_input(self.cfg, discourse, essay)\n        \n        return inputs\n        \nclass FeedBackModel(nn.Module):\n    def __init__(self, model_path):\n        super(FeedBackModel, self).__init__()\n        self.model = AutoModel.from_pretrained(model_path)\n        self.linear = nn.Linear(768, 3)\n\n    def forward(self, inputs):\n        last_hidden_states = self.model(**inputs)[0][:, 0, :]\n        outputs = self.linear(last_hidden_states)\n        \n        return outputs","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:09:27.403758Z","iopub.execute_input":"2022-07-12T05:09:27.404413Z","iopub.status.idle":"2022-07-12T05:09:27.414461Z","shell.execute_reply.started":"2022-07-12T05:09:27.404350Z","shell.execute_reply":"2022-07-12T05:09:27.413236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_list = pickle.load(\n    open(\"../input/feedback-roberta-ep1/roberta_modellist_ep2.pkl\", \"rb\")\n)\n\nclass CFG:\n    path = \"../input/roberta-base/\"\n    n_fold = 5\n    batch = 16\n    max_len = 512\n    num_workers = 2\n    \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.path)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:09:27.416145Z","iopub.execute_input":"2022-07-12T05:09:27.416620Z","iopub.status.idle":"2022-07-12T05:09:53.150997Z","shell.execute_reply.started":"2022-07-12T05:09:27.416579Z","shell.execute_reply":"2022-07-12T05:09:53.149894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test_origin.copy()\n\ntxt_sep = \" \"\ndf['discourse'] = df['discourse_type'].str.lower().str.strip() + txt_sep \\\n                + df['discourse_text'].str.lower().str.strip()\n\ndf['essay'] = df['essay_id'].transform(fetch_essay, txt_dir='test').str.lower().str.strip()\ndf.head()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:09:53.152803Z","iopub.execute_input":"2022-07-12T05:09:53.153450Z","iopub.status.idle":"2022-07-12T05:09:53.180555Z","shell.execute_reply.started":"2022-07-12T05:09:53.153405Z","shell.execute_reply":"2022-07-12T05:09:53.179408Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(CFG, df)\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch,\n                         shuffle=False, num_workers=CFG.num_workers,\n                         pin_memory=True, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:09:53.183459Z","iopub.execute_input":"2022-07-12T05:09:53.184138Z","iopub.status.idle":"2022-07-12T05:09:53.190494Z","shell.execute_reply.started":"2022-07-12T05:09:53.184096Z","shell.execute_reply":"2022-07-12T05:09:53.189421Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roberta_predicts = []\nfor i in range(CFG.n_fold):\n    model = model_list[i]\n    \n    prediction = inference_fn(test_loader, model, DEVICE)\n    roberta_predicts.append(prediction)\n    \n    del model, prediction\n    torch.cuda.empty_cache()    \n    gc.collect()\n    \ndel model_list\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:09:53.192166Z","iopub.execute_input":"2022-07-12T05:09:53.192641Z","iopub.status.idle":"2022-07-12T05:09:56.983407Z","shell.execute_reply.started":"2022-07-12T05:09:53.192597Z","shell.execute_reply":"2022-07-12T05:09:56.982407Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_ineffective = []\nrob_effective = []\nrob_adequate = []\n\nfor x in roberta_predicts:\n    rob_ineffective.append(x[:, 0])\n    rob_adequate.append(x[:, 1])\n    rob_effective.append(x[:, 2])\n\n# list -> dataframe\nrob_ineffective = pd.DataFrame(rob_ineffective).T\nrob_adequate = pd.DataFrame(rob_adequate).T\nrob_effective = pd.DataFrame(rob_effective).T","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:09:56.985303Z","iopub.execute_input":"2022-07-12T05:09:56.986018Z","iopub.status.idle":"2022-07-12T05:09:56.997746Z","shell.execute_reply.started":"2022-07-12T05:09:56.985976Z","shell.execute_reply":"2022-07-12T05:09:56.996420Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"<br>\n<h1 style = \"font-size:60px; font-family:Garamond ; font-weight : normal; background-color: #f6f5f5 ; color : #fe346e; text-align: center; border-radius: 100px 100px;\">Deberta-V3-Large (Tez)</h1>\n<br>","metadata":{}},{"cell_type":"code","source":"!cp -r ../input/tez-lib/ .\n!cd tez-lib && pip install .","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:09:56.999161Z","iopub.execute_input":"2022-07-12T05:09:57.001091Z","iopub.status.idle":"2022-07-12T05:10:31.980885Z","shell.execute_reply.started":"2022-07-12T05:09:57.001048Z","shell.execute_reply":"2022-07-12T05:10:31.979688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!cp ../input/fb2debertav3large/*.py .","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:10:31.982688Z","iopub.execute_input":"2022-07-12T05:10:31.983092Z","iopub.status.idle":"2022-07-12T05:10:32.801751Z","shell.execute_reply.started":"2022-07-12T05:10:31.983050Z","shell.execute_reply":"2022-07-12T05:10:32.800144Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!python main.py \\\n--model ../input/deberta-v3-large/deberta-v3-large \\\n--output ../input/fb2debertav3large \\\n--input ../input/feedback-prize-effectiveness/ \\\n--batch_size 2 \\\n--fold 0 \\\n--predict\n\n!python main.py \\\n--model ../input/deberta-v3-large/deberta-v3-large \\\n--output ../input/fb2debertav3large \\\n--input ../input/feedback-prize-effectiveness/ \\\n--batch_size 2 \\\n--fold 1 \\\n--predict\n\n!python main.py \\\n--model ../input/deberta-v3-large/deberta-v3-large \\\n--output ../input/fb2debertav3large \\\n--input ../input/feedback-prize-effectiveness/ \\\n--batch_size 2 \\\n--fold 2 \\\n--predict\n\n!python main.py \\\n--model ../input/deberta-v3-large/deberta-v3-large \\\n--output ../input/fb2debertav3large \\\n--input ../input/feedback-prize-effectiveness/ \\\n--batch_size 2 \\\n--fold 3 \\\n--predict\n\n!python main.py \\\n--model ../input/deberta-v3-large/deberta-v3-large \\\n--output ../input/fb2debertav3large \\\n--input ../input/feedback-prize-effectiveness/ \\\n--batch_size 2 \\\n--fold 4 \\\n--predict","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:10:32.803756Z","iopub.execute_input":"2022-07-12T05:10:32.804095Z","iopub.status.idle":"2022-07-12T05:14:14.811951Z","shell.execute_reply.started":"2022-07-12T05:10:32.804062Z","shell.execute_reply":"2022-07-12T05:14:14.810412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import glob\nimport pandas as pd\n\ndbv3_large_ineffective = []\ndbv3_large_effective = []\ndbv3_large_adequate = []\n\ncsvs = glob.glob(\"*.csv\")\nidx = []\npreds = []\nfor csv_idx, csv in enumerate(csvs):\n    df = pd.read_csv(csv)\n    print(df.head())\n    temp_preds = df.drop([\"discourse_id\"], axis=1).values\n    dbv3_large_ineffective.append(temp_preds[:, 0])\n    dbv3_large_adequate.append(temp_preds[:, 1])\n    dbv3_large_effective.append(temp_preds[:, 2])\n# list -> dataframe\ndbv3_large_ineffective = pd.DataFrame(dbv3_large_ineffective).T\ndbv3_large_adequate = pd.DataFrame(dbv3_large_adequate).T\ndbv3_large_effective = pd.DataFrame(dbv3_large_effective).T\n# print(dbv3_large_ineffective)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:14:14.814833Z","iopub.execute_input":"2022-07-12T05:14:14.815131Z","iopub.status.idle":"2022-07-12T05:14:14.860823Z","shell.execute_reply.started":"2022-07-12T05:14:14.815103Z","shell.execute_reply":"2022-07-12T05:14:14.859598Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_gradient(\n    dbv3_large_ineffective,\n    N_ROW)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:14:14.862636Z","iopub.execute_input":"2022-07-12T05:14:14.863052Z","iopub.status.idle":"2022-07-12T05:14:14.955449Z","shell.execute_reply.started":"2022-07-12T05:14:14.863009Z","shell.execute_reply":"2022-07-12T05:14:14.954355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Calculate the mean prediction probabilities of each folds\nsubmission = pd.read_csv('../input/feedback-prize-effectiveness/sample_submission.csv')\n\nlevel_names = ['deberta', 'deberta_large', 'deberta_v3_large']\n\nineffective_ = pd.concat(\n    [deb_ineffective, deb_large_ineffective, dbv3_large_ineffective],\n    keys=level_names, axis=1\n)\n\nadequate_ = pd.concat(\n    [deb_adequate,deb_large_adequate, dbv3_large_adequate],\n    keys=level_names, axis=1\n)\n\neffective_ = pd.concat(\n    [deb_effective, deb_large_effective, dbv3_large_effective],\n    keys=level_names, axis=1\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:14:14.957131Z","iopub.execute_input":"2022-07-12T05:14:14.957508Z","iopub.status.idle":"2022-07-12T05:14:14.974903Z","shell.execute_reply.started":"2022-07-12T05:14:14.957478Z","shell.execute_reply":"2022-07-12T05:14:14.973863Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_gradient(\n    ineffective_,\n    N_ROW\n)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:21:37.650172Z","iopub.execute_input":"2022-07-12T05:21:37.650767Z","iopub.status.idle":"2022-07-12T05:21:37.744669Z","shell.execute_reply.started":"2022-07-12T05:21:37.650732Z","shell.execute_reply":"2022-07-12T05:21:37.743476Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w_ = [0.05, 0.6, 0.35]  # ['deberta_base', 'deberta_large', 'deberta_v3_large']\nd_ = [('Ineffective', ineffective_),\n      ('Adequate', adequate_),\n      ('Effective', effective_)]\n\nfor x in d_:\n    col_name, df = x\n    submission[col_name] = pd.DataFrame(\n        {col: df[col].mean(axis=1) for col in level_names}\n    ).mul(w_).sum(axis=1)    \n\nsubmission.head(N_ROW)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:14:15.079506Z","iopub.execute_input":"2022-07-12T05:14:15.079966Z","iopub.status.idle":"2022-07-12T05:14:15.117571Z","shell.execute_reply.started":"2022-07-12T05:14:15.079925Z","shell.execute_reply":"2022-07-12T05:14:15.116573Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-07-12T05:14:15.119294Z","iopub.execute_input":"2022-07-12T05:14:15.120101Z","iopub.status.idle":"2022-07-12T05:14:15.127108Z","shell.execute_reply.started":"2022-07-12T05:14:15.120054Z","shell.execute_reply":"2022-07-12T05:14:15.126069Z"},"trusted":true},"execution_count":null,"outputs":[]}]}