{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"I have used almost all of the code present in below notebook with addition of LGBM(also mentioned below)\n\nFORK+ | Ensemble: deberta + roberta / Score: 0.614\n> https://www.kaggle.com/code/renokan/fork-ensemble-deberta-roberta\n\n## Ensemble of models\n\n1. feedback_deberta_large_LB0.619 / Score: 0.619\n> https://www.kaggle.com/code/brandonhu0215/feedback-deberta-large-lb0-619\n\n2. RoBerta-base Inference v2.0 / Score: 0.649\n> https://www.kaggle.com/code/arvissu/roberta-base-inference-v2-0\n\n3. Feedback2 - Word2vec + LightGBM\n> https://www.kaggle.com/code/mujrush/feedback2-word2vec-lightgbm/notebook\n\n#### If this notebook is helpful, please upvote the original versions:\n\n\n## Content\n\n> I tried to get the same result for each model, but by unifying the actions and removing the excess.\n>  \n> For these models, I use the same prepare_input and inference_fn.\n\n```\ndef prepare_input(cfg, text, text_2=None):\n    inputs = cfg.tokenizer(text, text_2,\n                           padding=\"max_length\",\n                           add_special_tokens=True,\n                           max_length=cfg.max_len,\n                           truncation=True)\n\n    [...]\n\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    \n    for inputs in tk0:\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n            \n        with torch.no_grad():\n            output = model(inputs)\n        \n    [...]\n```\n","metadata":{"papermill":{"duration":0.017223,"end_time":"2022-06-29T02:53:47.264285","exception":false,"start_time":"2022-06-29T02:53:47.247062","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"# 1. Import & Def & Set & Load","metadata":{"papermill":{"duration":0.013915,"end_time":"2022-06-29T02:53:47.292714","exception":false,"start_time":"2022-06-29T02:53:47.278799","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import gc\nimport os\nimport pickle\nimport glob\n\nfrom text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\n\nimport seaborn as sns\n\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\n\nfrom torch.nn import Parameter\nfrom torch.utils.data import Dataset, DataLoader\n\nfrom transformers import AutoModel, AutoTokenizer, AutoConfig\n\nimport warnings\nwarnings.simplefilter('ignore')","metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_kg_hide-input":true,"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","papermill":{"duration":8.806745,"end_time":"2022-06-29T02:53:56.118166","exception":false,"start_time":"2022-06-29T02:53:47.311421","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:04.445838Z","iopub.execute_input":"2022-07-07T11:44:04.446830Z","iopub.status.idle":"2022-07-07T11:44:10.888337Z","shell.execute_reply.started":"2022-07-07T11:44:04.446715Z","shell.execute_reply":"2022-07-07T11:44:10.887535Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    \n    text = unidecode(text)\n    \n    return text\n\n\ndef fetch_essay(essay_id: str, txt_dir: str):\n    essay_path = os.path.join(COMP_DIR + txt_dir, essay_id + '.txt')\n    essay_text = open(essay_path, 'r').read()\n    \n    return essay_text\n\n\ndef prepare_input(cfg, text, text_2=None):\n    inputs = cfg.tokenizer(text, text_2,\n                           padding=\"max_length\",\n                           add_special_tokens=True,\n                           max_length=cfg.max_len,\n                           truncation=True)\n\n    for k, v in inputs.items():\n        inputs[k] = torch.tensor(v, dtype=torch.long)\n        \n    return inputs\n\n\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    \n    for inputs in tk0:\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n            \n        with torch.no_grad():\n            output = model(inputs)\n        \n        preds.append(F.softmax(output).to('cpu').numpy())\n\n    return np.concatenate(preds)  \n\n\ndef show_gradient(df, n_row=None):\n    if not n_row:\n        n_row = 5\n\n    return df.head(n_row) \\\n                .assign(all_mean=lambda x: x.mean(axis=1)) \\\n                    .style.background_gradient(cmap=cm, axis=1)","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.02732,"end_time":"2022-06-29T02:53:56.154308","exception":false,"start_time":"2022-06-29T02:53:56.126988","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:10.890641Z","iopub.execute_input":"2022-07-07T11:44:10.891685Z","iopub.status.idle":"2022-07-07T11:44:10.906215Z","shell.execute_reply.started":"2022-07-07T11:44:10.891645Z","shell.execute_reply":"2022-07-07T11:44:10.905326Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.set_option('display.precision', 4)\ncm = sns.light_palette('green', as_cmap=True)\nprops_param = \"color:white; font-weight:bold; background-color:green;\"\n\nN_ROW = 10\n\nCOMP_DIR = \"../input/feedback-prize-effectiveness/\"\nDEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"papermill":{"duration":0.08735,"end_time":"2022-06-29T02:53:56.250659","exception":false,"start_time":"2022-06-29T02:53:56.163309","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:10.907525Z","iopub.execute_input":"2022-07-07T11:44:10.908076Z","iopub.status.idle":"2022-07-07T11:44:10.989944Z","shell.execute_reply.started":"2022-07-07T11:44:10.908039Z","shell.execute_reply":"2022-07-07T11:44:10.988976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_path = COMP_DIR + \"test.csv\"\nsubmission_path = COMP_DIR + \"sample_submission.csv\"\n\ntest_origin = pd.read_csv(test_path)\nsubmission_origin = pd.read_csv(submission_path)","metadata":{"papermill":{"duration":0.030936,"end_time":"2022-06-29T02:53:56.290445","exception":false,"start_time":"2022-06-29T02:53:56.259509","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:10.992573Z","iopub.execute_input":"2022-07-07T11:44:10.993833Z","iopub.status.idle":"2022-07-07T11:44:11.014139Z","shell.execute_reply.started":"2022-07-07T11:44:10.993784Z","shell.execute_reply":"2022-07-07T11:44:11.013442Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_origin.head()","metadata":{"papermill":{"duration":0.027815,"end_time":"2022-06-29T02:53:56.326953","exception":false,"start_time":"2022-06-29T02:53:56.299138","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:11.015222Z","iopub.execute_input":"2022-07-07T11:44:11.015601Z","iopub.status.idle":"2022-07-07T11:44:11.033378Z","shell.execute_reply.started":"2022-07-07T11:44:11.015555Z","shell.execute_reply":"2022-07-07T11:44:11.032669Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Check unidecode(text)\n\n```\ndef resolve_encodings_and_normalize(text: str) -> str:\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    \n    text = unidecode(text)\n```","metadata":{}},{"cell_type":"code","source":"data_path = \"../input/feedback-prize-effectiveness/train.csv\"\ncols_list = ['essay_id', 'discourse_text']\nidxs_list = [49, 80, 945, 947, 1870]\n\ntemp = pd.read_csv(data_path, usecols=cols_list).loc[idxs_list, :]\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:44:11.034527Z","iopub.execute_input":"2022-07-07T11:44:11.035005Z","iopub.status.idle":"2022-07-07T11:44:11.310517Z","shell.execute_reply.started":"2022-07-07T11:44:11.034969Z","shell.execute_reply":"2022-07-07T11:44:11.309648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"temp['discourse_text_UPD'] = temp['discourse_text'].apply(resolve_encodings_and_normalize)\n\ntemp['essay_text'] = temp['essay_id'].transform(fetch_essay, txt_dir='train')\ntemp['essay_text_UPD'] = temp['essay_text'].apply(resolve_encodings_and_normalize)\n\ntemp","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:44:11.311745Z","iopub.execute_input":"2022-07-07T11:44:11.312496Z","iopub.status.idle":"2022-07-07T11:44:11.351305Z","shell.execute_reply.started":"2022-07-07T11:44:11.312450Z","shell.execute_reply":"2022-07-07T11:44:11.350432Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for n, row in enumerate(temp.iterrows()):\n    indx, data = row\n    disc_text = data.discourse_text\n    disc_text_upd = data.discourse_text_UPD\n\n    print(f'\\nN{n} === index: {indx} ===')\n    print(f'\\n>>> origin text:')\n    print(repr(disc_text))\n    print(f'\\n>>> updated text:')\n    print(repr(disc_text_upd))","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:44:11.352819Z","iopub.execute_input":"2022-07-07T11:44:11.353178Z","iopub.status.idle":"2022-07-07T11:44:11.360921Z","shell.execute_reply.started":"2022-07-07T11:44:11.353142Z","shell.execute_reply":"2022-07-07T11:44:11.360117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Extract predictions","metadata":{"papermill":{"duration":0.00859,"end_time":"2022-06-29T02:53:56.344023","exception":false,"start_time":"2022-06-29T02:53:56.335433","status":"completed"},"tags":[]}},{"cell_type":"markdown","source":"## 3.1 DeBerta","metadata":{"papermill":{"duration":0.008456,"end_time":"2022-06-29T02:53:56.361487","exception":false,"start_time":"2022-06-29T02:53:56.353031","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.text = df['text'].values\n\n    def __len__(self):\n        return len(self.text)\n\n    def __getitem__(self, item):    \n        text = self.text[item]\n        inputs = prepare_input(self.cfg, text)\n        \n        return inputs\n\nclass CustomModel(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg\n        \n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n        else:\n            self.model = AutoModel.from_config(self.config)\n        \n        self.bilstm = nn.LSTM(self.config.hidden_size, (self.config.hidden_size) // 2, num_layers=2, \n                              dropout=self.config.hidden_dropout_prob, batch_first=True,\n                              bidirectional=True)\n        \n        # self.dropout = nn.Dropout(0.2)\n        self.dropout1 = nn.Dropout(0.1)\n        self.dropout2 = nn.Dropout(0.2)\n        self.dropout3 = nn.Dropout(0.3)\n        self.dropout4 = nn.Dropout(0.4)\n        self.dropout5 = nn.Dropout(0.5)\n        \n        self.output = nn.Sequential(\n            nn.Linear(self.config.hidden_size, 3)  # self.cfg.target_size\n        )\n                \n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.Embedding):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n\n    def forward(self, inputs):\n        sequence_output = self.model(**inputs)[0][:, 0, :]\n\n        logits1 = self.output(self.dropout1(sequence_output))\n        logits2 = self.output(self.dropout2(sequence_output))\n        logits3 = self.output(self.dropout3(sequence_output))\n        logits4 = self.output(self.dropout4(sequence_output))\n        logits5 = self.output(self.dropout5(sequence_output))\n        logits = (logits1 + logits2 + logits3 + logits4 + logits5) / 5\n\n        return logits","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.029514,"end_time":"2022-06-29T02:53:56.401394","exception":false,"start_time":"2022-06-29T02:53:56.37188","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:11.362541Z","iopub.execute_input":"2022-07-07T11:44:11.363330Z","iopub.status.idle":"2022-07-07T11:44:11.391648Z","shell.execute_reply.started":"2022-07-07T11:44:11.363292Z","shell.execute_reply":"2022-07-07T11:44:11.390236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    path = \"../input/feedback-deberta-large-051/\"\n    config_path = path+'config.pth'\n    model = \"microsoft/deberta-large\"\n    num_workers = 2\n    batch_size = 16\n    max_len = 512\n    seed = 42\n    n_fold = 4\n    # trn_fold = [0, 1, 2, 3]\n    # fc_dropout = 0.2\n    # target_size = 3\n    \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.path + 'tokenizer')","metadata":{"papermill":{"duration":0.182985,"end_time":"2022-06-29T02:53:56.593152","exception":false,"start_time":"2022-06-29T02:53:56.410167","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:11.401819Z","iopub.execute_input":"2022-07-07T11:44:11.402288Z","iopub.status.idle":"2022-07-07T11:44:11.567490Z","shell.execute_reply.started":"2022-07-07T11:44:11.402239Z","shell.execute_reply":"2022-07-07T11:44:11.566610Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test_origin.copy()\nSEP = CFG.tokenizer.sep_token\n\ndf['discourse_text'] = df['discourse_text'].apply(resolve_encodings_and_normalize)\ndf['essay_text'] = df['essay_id'].transform(fetch_essay, txt_dir='test')\ndf['essay_text'] = df['essay_text'].apply(resolve_encodings_and_normalize)\ndf['text'] = df['discourse_type'] + ' ' + df['discourse_text'] + SEP + df['essay_text']\n\ndf.head()","metadata":{"papermill":{"duration":0.054662,"end_time":"2022-06-29T02:53:56.657806","exception":false,"start_time":"2022-06-29T02:53:56.603144","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:11.568941Z","iopub.execute_input":"2022-07-07T11:44:11.569337Z","iopub.status.idle":"2022-07-07T11:44:11.600154Z","shell.execute_reply.started":"2022-07-07T11:44:11.569298Z","shell.execute_reply":"2022-07-07T11:44:11.599236Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(CFG, df)\ntest_loader = DataLoader(test_dataset,\n                         batch_size=CFG.batch_size,\n                         shuffle=False,\n                         num_workers=CFG.num_workers,\n                         pin_memory=True, drop_last=False)","metadata":{"papermill":{"duration":0.017801,"end_time":"2022-06-29T02:53:56.685024","exception":false,"start_time":"2022-06-29T02:53:56.667223","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:11.601521Z","iopub.execute_input":"2022-07-07T11:44:11.601864Z","iopub.status.idle":"2022-07-07T11:44:11.607106Z","shell.execute_reply.started":"2022-07-07T11:44:11.601829Z","shell.execute_reply":"2022-07-07T11:44:11.606042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deberta_predictions = []\n\nfor fold in range(CFG.n_fold):\n    model = CustomModel(CFG, config_path=CFG.config_path, pretrained=False)\n    state = torch.load(CFG.path+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\",\n                       map_location=torch.device('cpu'))\n    \n    model.load_state_dict(state['model'])\n    prediction = inference_fn(test_loader, model, DEVICE)\n    \n    deberta_predictions.append(prediction)\n    \n    del model, state, prediction; gc.collect()\n    torch.cuda.empty_cache()","metadata":{"papermill":{"duration":106.455245,"end_time":"2022-06-29T02:55:43.149193","exception":false,"start_time":"2022-06-29T02:53:56.693948","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:44:11.608847Z","iopub.execute_input":"2022-07-07T11:44:11.609274Z","iopub.status.idle":"2022-07-07T11:45:55.127353Z","shell.execute_reply.started":"2022-07-07T11:44:11.609241Z","shell.execute_reply":"2022-07-07T11:45:55.126458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_ineffective = []\ndeb_effective = []\ndeb_adequate = []\n\nfor x in deberta_predictions:\n    deb_ineffective.append(x[:, 0])\n    deb_adequate.append(x[:, 1])\n    deb_effective.append(x[:, 2])","metadata":{"papermill":{"duration":0.018775,"end_time":"2022-06-29T02:55:43.178719","exception":false,"start_time":"2022-06-29T02:55:43.159944","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:45:55.132034Z","iopub.execute_input":"2022-07-07T11:45:55.134163Z","iopub.status.idle":"2022-07-07T11:45:55.141225Z","shell.execute_reply.started":"2022-07-07T11:45:55.134119Z","shell.execute_reply":"2022-07-07T11:45:55.140602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_ineffective = pd.DataFrame(deb_ineffective).T\n\nshow_gradient(\n    deb_ineffective,\n    N_ROW)","metadata":{"papermill":{"duration":0.148585,"end_time":"2022-06-29T02:55:43.337415","exception":false,"start_time":"2022-06-29T02:55:43.18883","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:45:55.145104Z","iopub.execute_input":"2022-07-07T11:45:55.147162Z","iopub.status.idle":"2022-07-07T11:45:55.273998Z","shell.execute_reply.started":"2022-07-07T11:45:55.147126Z","shell.execute_reply":"2022-07-07T11:45:55.273313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_adequate = pd.DataFrame(deb_adequate).T\n\nshow_gradient(\n    deb_adequate,\n    N_ROW)","metadata":{"papermill":{"duration":0.040998,"end_time":"2022-06-29T02:55:43.389216","exception":false,"start_time":"2022-06-29T02:55:43.348218","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:45:55.277815Z","iopub.execute_input":"2022-07-07T11:45:55.279814Z","iopub.status.idle":"2022-07-07T11:45:55.314961Z","shell.execute_reply.started":"2022-07-07T11:45:55.279777Z","shell.execute_reply":"2022-07-07T11:45:55.314340Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deb_effective = pd.DataFrame(deb_effective).T\n\nshow_gradient(\n    deb_effective,\n    N_ROW)","metadata":{"papermill":{"duration":0.043433,"end_time":"2022-06-29T02:55:43.443852","exception":false,"start_time":"2022-06-29T02:55:43.400419","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:45:55.318647Z","iopub.execute_input":"2022-07-07T11:45:55.320659Z","iopub.status.idle":"2022-07-07T11:45:55.354964Z","shell.execute_reply.started":"2022-07-07T11:45:55.320624Z","shell.execute_reply":"2022-07-07T11:45:55.354343Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## 3.2 RoBerta","metadata":{"papermill":{"duration":0.011395,"end_time":"2022-06-29T02:55:43.467154","exception":false,"start_time":"2022-06-29T02:55:43.455759","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class TestDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.discourse = df['discourse'].values\n        self.essay = df['essay'].values\n        \n    def __len__(self):\n        return len(self.discourse)\n    \n    def __getitem__(self, item):\n        discourse = self.discourse[item]\n        essay = self.essay[item]\n        \n        inputs = prepare_input(self.cfg, discourse, essay)\n        \n        return inputs\n        \nclass FeedBackModel(nn.Module):\n    def __init__(self, model_path):\n        super(FeedBackModel, self).__init__()\n        self.model = AutoModel.from_pretrained(model_path)\n        self.linear = nn.Linear(768, 3)\n\n    def forward(self, inputs):\n        last_hidden_states = self.model(**inputs)[0][:, 0, :]\n        outputs = self.linear(last_hidden_states)\n        \n        return outputs","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.023062,"end_time":"2022-06-29T02:55:43.501615","exception":false,"start_time":"2022-06-29T02:55:43.478553","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:45:55.358686Z","iopub.execute_input":"2022-07-07T11:45:55.360855Z","iopub.status.idle":"2022-07-07T11:45:55.372950Z","shell.execute_reply.started":"2022-07-07T11:45:55.360819Z","shell.execute_reply":"2022-07-07T11:45:55.372072Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model_list = pickle.load(\n    open(\"../input/feedback-roberta-ep1/roberta_modellist_ep2.pkl\", \"rb\")\n)\n\nclass CFG:\n    path = \"../input/roberta-base/\"\n    n_fold = 5\n    batch = 16\n    max_len = 512\n    num_workers = 2\n    \nCFG.tokenizer = AutoTokenizer.from_pretrained(CFG.path)","metadata":{"papermill":{"duration":23.005949,"end_time":"2022-06-29T02:56:06.518411","exception":false,"start_time":"2022-06-29T02:55:43.512462","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:45:55.376835Z","iopub.execute_input":"2022-07-07T11:45:55.379143Z","iopub.status.idle":"2022-07-07T11:46:16.673239Z","shell.execute_reply.started":"2022-07-07T11:45:55.379106Z","shell.execute_reply":"2022-07-07T11:46:16.672439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = test_origin.copy()\n\ntxt_sep = \" \"\ndf['discourse'] = df['discourse_type'].str.lower().str.strip() + txt_sep \\\n                + df['discourse_text'].str.lower().str.strip()\n\ndf['essay'] = df['essay_id'].transform(fetch_essay, txt_dir='test').str.lower().str.strip()\ndf.head()","metadata":{"papermill":{"duration":0.042065,"end_time":"2022-06-29T02:56:06.57446","exception":false,"start_time":"2022-06-29T02:56:06.532395","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:46:16.674532Z","iopub.execute_input":"2022-07-07T11:46:16.676801Z","iopub.status.idle":"2022-07-07T11:46:16.699739Z","shell.execute_reply.started":"2022-07-07T11:46:16.676771Z","shell.execute_reply":"2022-07-07T11:46:16.698901Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_dataset = TestDataset(CFG, df)\ntest_loader = DataLoader(test_dataset, batch_size=CFG.batch,\n                         shuffle=False, num_workers=CFG.num_workers,\n                         pin_memory=True, drop_last=False)","metadata":{"papermill":{"duration":0.019897,"end_time":"2022-06-29T02:56:06.606774","exception":false,"start_time":"2022-06-29T02:56:06.586877","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:46:16.701002Z","iopub.execute_input":"2022-07-07T11:46:16.701487Z","iopub.status.idle":"2022-07-07T11:46:16.706890Z","shell.execute_reply.started":"2022-07-07T11:46:16.701446Z","shell.execute_reply":"2022-07-07T11:46:16.706213Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"roberta_predicts = []\nfor i in range(CFG.n_fold):\n    model = model_list[i]\n    \n    prediction = inference_fn(test_loader, model, DEVICE)\n    roberta_predicts.append(prediction)\n    \n    del model, prediction\n    torch.cuda.empty_cache()    \n    gc.collect()\n    \ndel model_list\ngc.collect()","metadata":{"papermill":{"duration":3.524253,"end_time":"2022-06-29T02:56:10.143211","exception":false,"start_time":"2022-06-29T02:56:06.618958","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:46:16.708130Z","iopub.execute_input":"2022-07-07T11:46:16.709167Z","iopub.status.idle":"2022-07-07T11:46:20.019244Z","shell.execute_reply.started":"2022-07-07T11:46:16.709132Z","shell.execute_reply":"2022-07-07T11:46:20.018266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_ineffective = []\nrob_effective = []\nrob_adequate = []\n\nfor x in roberta_predicts:\n    rob_ineffective.append(x[:, 0])\n    rob_adequate.append(x[:, 1])\n    rob_effective.append(x[:, 2])","metadata":{"papermill":{"duration":0.022375,"end_time":"2022-06-29T02:56:10.178553","exception":false,"start_time":"2022-06-29T02:56:10.156178","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:46:20.020668Z","iopub.execute_input":"2022-07-07T11:46:20.021566Z","iopub.status.idle":"2022-07-07T11:46:20.027360Z","shell.execute_reply.started":"2022-07-07T11:46:20.021513Z","shell.execute_reply":"2022-07-07T11:46:20.026633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_ineffective = pd.DataFrame(rob_ineffective).T\n\nshow_gradient(\n    rob_ineffective,\n    N_ROW)","metadata":{"papermill":{"duration":0.07251,"end_time":"2022-06-29T02:56:10.263745","exception":false,"start_time":"2022-06-29T02:56:10.191235","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:46:20.028694Z","iopub.execute_input":"2022-07-07T11:46:20.029075Z","iopub.status.idle":"2022-07-07T11:46:20.064983Z","shell.execute_reply.started":"2022-07-07T11:46:20.029039Z","shell.execute_reply":"2022-07-07T11:46:20.064250Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_adequate = pd.DataFrame(rob_adequate).T\n\nshow_gradient(\n    rob_adequate,\n    N_ROW)","metadata":{"papermill":{"duration":0.095428,"end_time":"2022-06-29T02:56:10.3919","exception":false,"start_time":"2022-06-29T02:56:10.296472","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:46:20.066381Z","iopub.execute_input":"2022-07-07T11:46:20.066797Z","iopub.status.idle":"2022-07-07T11:46:20.095839Z","shell.execute_reply.started":"2022-07-07T11:46:20.066755Z","shell.execute_reply":"2022-07-07T11:46:20.094923Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"rob_effective = pd.DataFrame(rob_effective).T\n\nshow_gradient(\n    rob_effective,\n    N_ROW)","metadata":{"papermill":{"duration":0.081635,"end_time":"2022-06-29T02:56:10.511529","exception":false,"start_time":"2022-06-29T02:56:10.429894","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:46:20.096963Z","iopub.execute_input":"2022-07-07T11:46:20.097363Z","iopub.status.idle":"2022-07-07T11:46:20.127768Z","shell.execute_reply.started":"2022-07-07T11:46:20.097328Z","shell.execute_reply":"2022-07-07T11:46:20.127013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LGBM","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.preprocessing import OneHotEncoder\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom sklearn.metrics import log_loss\n\nimport gensim\nfrom scipy import sparse\nimport lightgbm as lgb\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:46:20.129150Z","iopub.execute_input":"2022-07-07T11:46:20.129605Z","iopub.status.idle":"2022-07-07T11:46:22.323713Z","shell.execute_reply.started":"2022-07-07T11:46:20.129566Z","shell.execute_reply":"2022-07-07T11:46:22.322784Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# class CFG:\n#     seed = 42\n#     n_folds = 4\n    \n# INPUT_DIR = \"../input/feedback-prize-effectiveness/\"\n\n# def get_train_essay(essay_id):\n#     essay_path = os.path.join(INPUT_DIR,f'train/{essay_id}.txt')\n#     essay_text = open(essay_path,'r').read()\n#     return essay_text\n\n# def get_test_essay(essay_id):\n#     essay_path = os.path.join(INPUT_DIR,f'test/{essay_id}.txt')\n#     essay_text = open(essay_path,'r').read()\n#     return essay_text\n\n# train = pd.read_csv(INPUT_DIR+'train.csv')\n# test = pd.read_csv(INPUT_DIR+'test.csv')\n# train['essay_text'] = train['essay_id'].apply(get_train_essay)\n# test['essay_text'] = test['essay_id'].apply(get_test_essay)\n\n# def set_seed(seed=42):\n#     np.random.seed(seed)\n#     os.environ['PYTHONHASHSEED'] = str(seed)\n\n# set_seed(CFG.seed)\n\n# effectiveness_map = {'Ineffective':0, 'Adequate':1, 'Effective':2}\n# train['target'] = train['discourse_effectiveness'].map(effectiveness_map)\n\n# sgkf = StratifiedGroupKFold(n_splits=CFG.n_folds,shuffle=True,random_state=CFG.seed)\n\n# for fold, (_,val_idx) in enumerate(sgkf.split(X=train, y=train['target'], groups=train.essay_id)):\n#     train.loc[val_idx,'kfold'] = fold\n\n# word2vec_model = gensim.models.KeyedVectors.load_word2vec_format('../input/google-news/GoogleNews-vectors-negative300.bin', binary=True)\n# print(word2vec_model.vectors.shape)\n\n# def avg_feature_vector(sentence, model, num_features):\n#     words = sentence.replace('\\n',\" \").replace(',',' ').replace('.',\" \").split()\n#     feature_vec = np.zeros((num_features,),dtype=\"float32\")\n#     i=0\n#     for word in words:\n#         try:\n#             feature_vec = np.add(feature_vec, model[word])\n#         except KeyError as error:\n#             feature_vec \n#             i = i + 1\n#     if len(words) > 0:\n#         feature_vec = np.divide(feature_vec, len(words)- i)\n#     return feature_vec\n\n# params = {}\n# params[\"objective\"] = 'multiclass'\n# params['metric'] = 'multi_logloss'\n# params['boosting'] = 'gbdt'\n# params['num_class'] = 3\n# params['is_unbalance'] = True\n# params[\"learning_rate\"] = 0.05\n# params[\"lambda_l2\"] = 0.0256\n# params[\"num_leaves\"] = 52\n# params[\"max_depth\"] = 10\n# params[\"feature_fraction\"] = 0.503\n# params[\"bagging_fraction\"] = 0.741\n# params[\"bagging_freq\"] = 8\n# params[\"bagging_seed\"] = 10\n# params[\"min_data_in_leaf\"] = 10\n# params[\"verbosity\"] = -1\n# params[\"random_state\"] = 42\n# num_rounds = 1000\n\n# oof_score = 0\n# y_test_pred = np.zeros((test.shape[0], 3))\n\n# for fold in range(CFG.n_folds):\n#     print(f'=============fold:{fold}==================')\n#     train_fold=train[train['kfold']!=fold].reset_index(drop=True)\n#     valid_fold=train[train['kfold']==fold].reset_index(drop=True)\n\n#     #word2vec\n\n#     #discourse_text\n#     word2vec_train_disc_text = np.zeros((len(train_fold.index),300),dtype=\"float32\")\n#     word2vec_valid_disc_text = np.zeros((len(valid_fold.index),300),dtype=\"float32\")\n#     word2vec_test_disc_text = np.zeros((len(test.index),300),dtype=\"float32\")\n#     for i in range(len(train_fold.index)):\n#         word2vec_train_disc_text[i] = avg_feature_vector(train_fold[\"discourse_text\"][i], word2vec_model, 300)\n#     for i in range(len(valid_fold.index)):\n#         word2vec_valid_disc_text[i] = avg_feature_vector(valid_fold[\"discourse_text\"][i], word2vec_model, 300)\n#     for i in range(len(test.index)):\n#         word2vec_test_disc_text[i] = avg_feature_vector(test[\"discourse_text\"][i], word2vec_model, 300)\n\n#     #essay_text\n#     word2vec_train_essay_text = np.zeros((len(train_fold.index),300),dtype=\"float32\")\n#     word2vec_valid_essay_text = np.zeros((len(valid_fold.index),300),dtype=\"float32\")\n#     word2vec_test_essay_text = np.zeros((len(test.index),300),dtype=\"float32\")\n#     for i in range(len(train_fold.index)):\n#         word2vec_train_essay_text[i] = avg_feature_vector(train_fold[\"essay_text\"][i], word2vec_model, 300)\n#     for i in range(len(valid_fold.index)):\n#         word2vec_valid_essay_text[i] = avg_feature_vector(valid_fold[\"essay_text\"][i], word2vec_model, 300)\n#     for i in range(len(test.index)):\n#         word2vec_test_essay_text[i] = avg_feature_vector(test[\"essay_text\"][i], word2vec_model, 300)\n\n#     #OneHot\n#     ohe = OneHotEncoder()\n#     train_type_ohe=sparse.csr_matrix(ohe.fit_transform(train_fold['discourse_type'].values.reshape(-1,1)))\n#     valid_type_ohe=sparse.csr_matrix(ohe.transform(valid_fold['discourse_type'].values.reshape(-1,1)))\n#     test_type_ohe=sparse.csr_matrix(ohe.transform(test['discourse_type'].values.reshape(-1,1)))\n\n\n#     #merge\n#     Xtrain_word2vec = sparse.hstack((train_type_ohe,word2vec_train_disc_text,word2vec_train_essay_text))\n#     Xvalid_word2vec = sparse.hstack((valid_type_ohe,word2vec_valid_disc_text,word2vec_valid_essay_text))\n#     test_word2vec = sparse.hstack((test_type_ohe,word2vec_test_disc_text,word2vec_test_essay_text))\n\n#     #lgbm\n#     lgtrain = lgb.Dataset(Xtrain_word2vec, label=train_fold['target'].ravel())\n#     lgvalidation = lgb.Dataset(Xvalid_word2vec, label=valid_fold['target'].ravel())\n\n#     model = lgb.train(params, lgtrain, num_rounds, \n#                     valid_sets=[lgtrain, lgvalidation], \n#                     early_stopping_rounds=100, verbose_eval=100)\n\n#     y_pred = model.predict(Xvalid_word2vec, num_iteration=model.best_iteration)\n#     y_test_pred += model.predict(test_word2vec, num_iteration=model.best_iteration)\n\n#     score = log_loss(valid_fold['target'], y_pred)\n#     oof_score += score\n\n#     print(f'Fold:{fold},valid score:{score}')\n    \n# y_test_pred = y_test_pred / float(CFG.n_folds)\n# oof_score /= float(CFG.n_folds)\n# print(\"Aggregate OOF Score: {}\".format(oof_score))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:46:22.328049Z","iopub.execute_input":"2022-07-07T11:46:22.328686Z","iopub.status.idle":"2022-07-07T11:46:22.338014Z","shell.execute_reply.started":"2022-07-07T11:46:22.328644Z","shell.execute_reply":"2022-07-07T11:46:22.336542Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n    seed = 42\n    n_folds = 4\n    \nINPUT_DIR = \"../input/feedback-prize-effectiveness/\"\n\ndef get_train_essay(essay_id):\n    essay_path = os.path.join(INPUT_DIR,f'train/{essay_id}.txt')\n    essay_text = open(essay_path,'r').read()\n    return essay_text\n\ndef get_test_essay(essay_id):\n    essay_path = os.path.join(INPUT_DIR,f'test/{essay_id}.txt')\n    essay_text = open(essay_path,'r').read()\n    return essay_text\n\ntrain = pd.read_csv(INPUT_DIR+'train.csv')\ntest = pd.read_csv(INPUT_DIR+'test.csv')\ntrain['essay_text'] = train['essay_id'].apply(get_train_essay)\ntest['essay_text'] = test['essay_id'].apply(get_test_essay)\n\ndef set_seed(seed=42):\n    np.random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n\nset_seed(CFG.seed)\n\neffectiveness_map = {'Ineffective':0, 'Adequate':1, 'Effective':2}\ntrain['target'] = train['discourse_effectiveness'].map(effectiveness_map)\n\nsgkf = StratifiedGroupKFold(n_splits=CFG.n_folds,shuffle=True,random_state=CFG.seed)\n\nfor fold, (_,val_idx) in enumerate(sgkf.split(X=train, y=train['target'], groups=train.essay_id)):\n    train.loc[val_idx,'kfold'] = fold\n\n# word2vec_model = gensim.models.KeyedVectors.load_word2vec_format('../input/google-news/GoogleNews-vectors-negative300.bin', binary=True)\n# print(word2vec_model.vectors.shape)\nwith open('../input/pickled-glove840b300d-for-10sec-loading/glove.840B.300d.pkl', 'rb') as fp:\n    word2vec_model = pickle.load(fp)\n\n    \ndef avg_feature_vector(sentence, model, num_features):\n    words = sentence.replace('\\n',\" \").replace(',',' ').replace('.',\" \").split()\n    feature_vec = np.zeros((num_features,),dtype=\"float32\")\n    i=0\n    for word in words:\n        try:\n            feature_vec = np.add(feature_vec, model[word])\n        except KeyError as error:\n            feature_vec \n            i = i + 1\n    if len(words) > 0:\n        feature_vec = np.divide(feature_vec, len(words)- i)\n    return feature_vec\n\nparams = {}\nparams[\"objective\"] = 'multiclass'\nparams['metric'] = 'multi_logloss'\nparams['boosting'] = 'gbdt'\nparams['num_class'] = 3\nparams['is_unbalance'] = True\nparams[\"learning_rate\"] = 0.05\nparams[\"lambda_l2\"] = 0.0256\nparams[\"num_leaves\"] = 52\nparams[\"max_depth\"] = 10\nparams[\"feature_fraction\"] = 0.503\nparams[\"bagging_fraction\"] = 0.741\nparams[\"bagging_freq\"] = 8\nparams[\"bagging_seed\"] = 10\nparams[\"min_data_in_leaf\"] = 10\nparams[\"verbosity\"] = -1\nparams[\"random_state\"] = 42\nnum_rounds = 1000\n\noof_score = 0\ny_test_pred = np.zeros((test.shape[0], 3))\n\nfor fold in range(CFG.n_folds):\n    print(f'=============fold:{fold}==================')\n    train_fold=train[train['kfold']!=fold].reset_index(drop=True)\n    valid_fold=train[train['kfold']==fold].reset_index(drop=True)\n\n    #word2vec\n\n    #discourse_text\n    word2vec_train_disc_text = np.zeros((len(train_fold.index),300),dtype=\"float32\")\n    word2vec_valid_disc_text = np.zeros((len(valid_fold.index),300),dtype=\"float32\")\n    word2vec_test_disc_text = np.zeros((len(test.index),300),dtype=\"float32\")\n    for i in range(len(train_fold.index)):\n        word2vec_train_disc_text[i] = avg_feature_vector(train_fold[\"discourse_text\"][i], word2vec_model, 300)\n    for i in range(len(valid_fold.index)):\n        word2vec_valid_disc_text[i] = avg_feature_vector(valid_fold[\"discourse_text\"][i], word2vec_model, 300)\n    for i in range(len(test.index)):\n        word2vec_test_disc_text[i] = avg_feature_vector(test[\"discourse_text\"][i], word2vec_model, 300)\n\n    #essay_text\n    word2vec_train_essay_text = np.zeros((len(train_fold.index),300),dtype=\"float32\")\n    word2vec_valid_essay_text = np.zeros((len(valid_fold.index),300),dtype=\"float32\")\n    word2vec_test_essay_text = np.zeros((len(test.index),300),dtype=\"float32\")\n    for i in range(len(train_fold.index)):\n        word2vec_train_essay_text[i] = avg_feature_vector(train_fold[\"essay_text\"][i], word2vec_model, 300)\n    for i in range(len(valid_fold.index)):\n        word2vec_valid_essay_text[i] = avg_feature_vector(valid_fold[\"essay_text\"][i], word2vec_model, 300)\n    for i in range(len(test.index)):\n        word2vec_test_essay_text[i] = avg_feature_vector(test[\"essay_text\"][i], word2vec_model, 300)\n\n    #OneHot\n    ohe = OneHotEncoder()\n    train_type_ohe=sparse.csr_matrix(ohe.fit_transform(train_fold['discourse_type'].values.reshape(-1,1)))\n    valid_type_ohe=sparse.csr_matrix(ohe.transform(valid_fold['discourse_type'].values.reshape(-1,1)))\n    test_type_ohe=sparse.csr_matrix(ohe.transform(test['discourse_type'].values.reshape(-1,1)))\n\n\n    #merge\n    Xtrain_word2vec = sparse.hstack((train_type_ohe,word2vec_train_disc_text,word2vec_train_essay_text))\n    Xvalid_word2vec = sparse.hstack((valid_type_ohe,word2vec_valid_disc_text,word2vec_valid_essay_text))\n    test_word2vec = sparse.hstack((test_type_ohe,word2vec_test_disc_text,word2vec_test_essay_text))\n\n    #lgbm\n    lgtrain = lgb.Dataset(Xtrain_word2vec, label=train_fold['target'].ravel())\n    lgvalidation = lgb.Dataset(Xvalid_word2vec, label=valid_fold['target'].ravel())\n\n    model = lgb.train(params, lgtrain, num_rounds, \n                    valid_sets=[lgtrain, lgvalidation], \n                    early_stopping_rounds=100, verbose_eval=100)\n\n    y_pred = model.predict(Xvalid_word2vec, num_iteration=model.best_iteration)\n    y_test_pred += model.predict(test_word2vec, num_iteration=model.best_iteration)\n\n    score = log_loss(valid_fold['target'], y_pred)\n    oof_score += score\n\n    print(f'Fold:{fold},valid score:{score}')\n    \ny_test_pred = y_test_pred / float(CFG.n_folds)\noof_score /= float(CFG.n_folds)\nprint(\"Aggregate OOF Score: {}\".format(oof_score))\n\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:46:22.339461Z","iopub.execute_input":"2022-07-07T11:46:22.339824Z","iopub.status.idle":"2022-07-07T11:58:08.149050Z","shell.execute_reply.started":"2022-07-07T11:46:22.339789Z","shell.execute_reply":"2022-07-07T11:58:08.148117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"lgbm_ineffective = y_test_pred[:,0]\nlgbm_adequate = y_test_pred[:,1]\nlgbm_effective = y_test_pred[:,2]\n\nlgbm_ineffective = pd.DataFrame(lgbm_ineffective)\nlgbm_adequate = pd.DataFrame(lgbm_adequate)\nlgbm_effective = pd.DataFrame(lgbm_effective)\n","metadata":{"execution":{"iopub.status.busy":"2022-07-07T11:58:08.150605Z","iopub.execute_input":"2022-07-07T11:58:08.150975Z","iopub.status.idle":"2022-07-07T11:58:08.156229Z","shell.execute_reply.started":"2022-07-07T11:58:08.150937Z","shell.execute_reply":"2022-07-07T11:58:08.155454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Create submission","metadata":{"papermill":{"duration":0.024077,"end_time":"2022-06-29T02:56:10.562061","exception":false,"start_time":"2022-06-29T02:56:10.537984","status":"completed"},"tags":[]}},{"cell_type":"code","source":"level_names = ['deberta', 'roberta', 'lgbm']\n\nineffective_ = pd.concat(\n    [deb_ineffective, rob_ineffective, lgbm_ineffective],\n    keys=level_names, axis=1\n)\n\nadequate_ = pd.concat(\n    [deb_adequate, rob_adequate, lgbm_adequate],\n    keys=level_names, axis=1\n)\n\neffective_ = pd.concat(\n    [deb_effective, rob_effective, lgbm_effective],\n    keys=level_names, axis=1\n)","metadata":{"papermill":{"duration":0.046338,"end_time":"2022-06-29T02:56:10.633434","exception":false,"start_time":"2022-06-29T02:56:10.587096","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:58:08.157660Z","iopub.execute_input":"2022-07-07T11:58:08.158239Z","iopub.status.idle":"2022-07-07T11:58:08.173646Z","shell.execute_reply.started":"2022-07-07T11:58:08.158202Z","shell.execute_reply":"2022-07-07T11:58:08.172774Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_gradient(\n    ineffective_,\n    N_ROW\n)","metadata":{"papermill":{"duration":0.137257,"end_time":"2022-06-29T02:56:10.79562","exception":false,"start_time":"2022-06-29T02:56:10.658363","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:58:08.174918Z","iopub.execute_input":"2022-07-07T11:58:08.175324Z","iopub.status.idle":"2022-07-07T11:58:08.254258Z","shell.execute_reply.started":"2022-07-07T11:58:08.175289Z","shell.execute_reply":"2022-07-07T11:58:08.253311Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_gradient(\n    adequate_,\n    N_ROW\n)","metadata":{"papermill":{"duration":0.102402,"end_time":"2022-06-29T02:56:10.923467","exception":false,"start_time":"2022-06-29T02:56:10.821065","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:58:08.255886Z","iopub.execute_input":"2022-07-07T11:58:08.256261Z","iopub.status.idle":"2022-07-07T11:58:08.322777Z","shell.execute_reply.started":"2022-07-07T11:58:08.256224Z","shell.execute_reply":"2022-07-07T11:58:08.321869Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_gradient(\n    effective_,\n    N_ROW\n)","metadata":{"papermill":{"duration":0.079449,"end_time":"2022-06-29T02:56:11.019182","exception":false,"start_time":"2022-06-29T02:56:10.939733","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:58:08.323985Z","iopub.execute_input":"2022-07-07T11:58:08.324475Z","iopub.status.idle":"2022-07-07T11:58:08.392716Z","shell.execute_reply.started":"2022-07-07T11:58:08.324434Z","shell.execute_reply":"2022-07-07T11:58:08.391835Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = submission_origin.copy()\n\nw_ = [.65, .25, .1]  # ['deberta', 'roberta', 'gbm']\nd_ = [('Ineffective', ineffective_),\n      ('Adequate', adequate_),\n      ('Effective', effective_)]\n\nfor x in d_:\n    col_name, df = x\n    submission[col_name] = pd.DataFrame(\n        {col: df[col].mean(axis=1) for col in level_names}\n    ).mul(w_).sum(axis=1)    \n\nsubmission.head(N_ROW)","metadata":{"papermill":{"duration":0.042962,"end_time":"2022-06-29T02:56:11.078725","exception":false,"start_time":"2022-06-29T02:56:11.035763","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:58:08.394170Z","iopub.execute_input":"2022-07-07T11:58:08.394571Z","iopub.status.idle":"2022-07-07T11:58:08.428777Z","shell.execute_reply.started":"2022-07-07T11:58:08.394531Z","shell.execute_reply":"2022-07-07T11:58:08.427762Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# 0\ta261b6e14276\t0.0102\t0.3887\t0.5911\n# 1\t5a88900e7dc1\t0.0309\t0.8405\t0.1185\n# 2\t9790d835736b\t0.0217\t0.6997\t0.2686\n# 3\t75ce6d68b67b\t0.0512\t0.6365\t0.3023\n# 4\t93578d946723\t0.0399\t0.6053\t0.3448\n# 5\t2e214524dbe3\t0.0099\t0.3721\t0.6080\n# 6\t84812fc2ab9f\t0.0084\t0.2796\t0.7020\n# 7\tc668ff840720\t0.0171\t0.5888\t0.3841\n# 8\t739a6d00f44a\t0.0178\t0.4030\t0.5692\n# 9\tbcfae2c9a244\t0.0132\t0.6373\t0.3395","metadata":{"_kg_hide-input":true,"papermill":{"duration":0.021768,"end_time":"2022-06-29T02:56:11.116966","exception":false,"start_time":"2022-06-29T02:56:11.095198","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T11:58:08.430534Z","iopub.execute_input":"2022-07-07T11:58:08.430908Z","iopub.status.idle":"2022-07-07T11:58:08.435359Z","shell.execute_reply.started":"2022-07-07T11:58:08.430871Z","shell.execute_reply":"2022-07-07T11:58:08.434303Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission['Ineffective'] = np.where(submission['Ineffective'] <0.1,0,submission['Ineffective'])\nsubmission['Adequate'] = np.where(submission['Adequate'] <0.1,0,submission['Adequate'])\nsubmission['Effective'] = np.where(submission['Effective'] <0.1,0,submission['Effective'])\n\n\nsubmission.to_csv('submission.csv',index=False)","metadata":{"papermill":{"duration":0.027941,"end_time":"2022-06-29T02:56:11.160924","exception":false,"start_time":"2022-06-29T02:56:11.132983","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-07-07T12:28:43.555873Z","iopub.execute_input":"2022-07-07T12:28:43.556225Z","iopub.status.idle":"2022-07-07T12:28:43.565144Z","shell.execute_reply.started":"2022-07-07T12:28:43.556197Z","shell.execute_reply":"2022-07-07T12:28:43.564085Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}