{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\ntrain_df = pd.read_csv(\"/kaggle/input/commonlitreadabilityprize/train.csv\")\ntest_df = pd.read_csv(\"/kaggle/input/commonlitreadabilityprize/test.csv\")\nsubmission_df = pd.read_csv(\"/kaggle/input/commonlitreadabilityprize/sample_submission.csv\")","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:29:32.086699Z","iopub.execute_input":"2021-08-07T10:29:32.087064Z","iopub.status.idle":"2021-08-07T10:29:32.177754Z","shell.execute_reply.started":"2021-08-07T10:29:32.086987Z","shell.execute_reply":"2021-08-07T10:29:32.176925Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 1","metadata":{}},{"cell_type":"code","source":"import gc","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:29:32.179104Z","iopub.execute_input":"2021-08-07T10:29:32.179442Z","iopub.status.idle":"2021-08-07T10:29:32.184569Z","shell.execute_reply.started":"2021-08-07T10:29:32.179406Z","shell.execute_reply":"2021-08-07T10:29:32.183554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import math\n# import random\n# import time\n\n# import numpy as np\n\n\n# import torch\n# import torch.nn as nn\n# from torch.utils.data import Dataset\n# from torch.utils.data import DataLoader\n\n# from transformers import AutoTokenizer\n# from transformers import AutoModel\n# from transformers import AutoConfig\n\n# from sklearn.model_selection import KFold\n# from sklearn.svm import SVR\n\n# import gc\n# gc.enable()\n\n# BATCH_SIZE = 32\n# MAX_LEN = 248\n# EVAL_SCHEDULE = [(0.50, 16), (0.49, 8), (0.48, 4), (0.47, 2), (-1., 1)]\n# # ROBERTA_PATH = \"/kaggle/input/roberta-base\"\n# # TOKENIZER_PATH = \"/kaggle/input/roberta-base\"\n# ROBERTA_PATH = \"../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-base\"\n# TOKENIZER_PATH = \"../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-base\"\n# DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH)\n\n# class LitDataset(Dataset):\n#     def __init__(self, df, inference_only=False):\n#         super().__init__()\n\n#         self.df = df        \n#         self.inference_only = inference_only\n#         self.text = df.excerpt.tolist()\n#         #self.text = [text.replace(\"\\n\", \" \") for text in self.text]\n        \n#         if not self.inference_only:\n#             self.target = torch.tensor(df.target.values, dtype=torch.float32)        \n    \n#         self.encoded = tokenizer.batch_encode_plus(\n#             self.text,\n#             padding = 'max_length',            \n#             max_length = MAX_LEN,\n#             truncation = True,\n#             return_attention_mask=True\n#         )        \n \n\n#     def __len__(self):\n#         return len(self.df)\n\n    \n#     def __getitem__(self, index):        \n#         input_ids = torch.tensor(self.encoded['input_ids'][index])\n#         attention_mask = torch.tensor(self.encoded['attention_mask'][index])\n        \n#         if self.inference_only:\n#             return (input_ids, attention_mask)            \n#         else:\n#             target = self.target[index]\n#             return (input_ids, attention_mask, target)\n        \n\n# class LitModel(nn.Module):\n#     def __init__(self):\n#         super().__init__()\n\n#         config = AutoConfig.from_pretrained(ROBERTA_PATH)\n#         config.update({\"output_hidden_states\":True, \n#                        \"hidden_dropout_prob\": 0.0,\n#                        \"layer_norm_eps\": 1e-7})                       \n        \n#         self.roberta = AutoModel.from_pretrained(ROBERTA_PATH, config=config)  \n            \n#         self.attention = nn.Sequential(            \n#             nn.Linear(768, 512),            \n#             nn.Tanh(),                       \n#             nn.Linear(512, 1),\n#             nn.Softmax(dim=1)\n#         )        \n\n#         self.regressor = nn.Sequential(                        \n#             nn.Linear(768, 1)                        \n#         )\n        \n\n#     def forward(self, input_ids, attention_mask):\n#         roberta_output = self.roberta(input_ids=input_ids,\n#                                       attention_mask=attention_mask)        \n\n#         # There are a total of 13 layers of hidden states.\n#         # 1 for the embedding layer, and 12 for the 12 Roberta layers.\n#         # We take the hidden states from the last Roberta layer.\n#         last_layer_hidden_states = roberta_output.hidden_states[-1]\n\n#         # The number of cells is MAX_LEN.\n#         # The size of the hidden state of each cell is 768 (for roberta-base).\n#         # In order to condense hidden states of all cells to a context vector,\n#         # we compute a weighted average of the hidden states of all cells.\n#         # We compute the weight of each cell, using the attention neural network.\n#         weights = self.attention(last_layer_hidden_states)\n                \n#         # weights.shape is BATCH_SIZE x MAX_LEN x 1\n#         # last_layer_hidden_states.shape is BATCH_SIZE x MAX_LEN x 768        \n#         # Now we compute context_vector as the weighted average.\n#         # context_vector.shape is BATCH_SIZE x 768\n#         context_vector = torch.sum(weights * last_layer_hidden_states, dim=1)        \n        \n#         # Now we reduce the context vector to the prediction score.\n#         return self.regressor(context_vector)\n\n# def predict(model, data_loader):\n#     \"\"\"Returns an np.array with predictions of the |model| on |data_loader|\"\"\"\n#     model.eval()\n\n#     result = np.zeros(len(data_loader.dataset))    \n#     index = 0\n    \n#     with torch.no_grad():\n#         for batch_num, (input_ids, attention_mask) in enumerate(data_loader):\n#             input_ids = input_ids.to(DEVICE)\n#             attention_mask = attention_mask.to(DEVICE)\n                        \n#             pred = model(input_ids, attention_mask)                        \n\n#             result[index : index + pred.shape[0]] = pred.flatten().to(\"cpu\")\n#             index += pred.shape[0]\n\n#     return result\n        \n# test_dataset = LitDataset(test_df, inference_only=True)\n# NUM_MODELS = 5\n\n# all_predictions = np.zeros((NUM_MODELS, len(test_df)))\n\n\n\n# test_dataset = LitDataset(test_df, inference_only=True)\n# test_loader = DataLoader(test_dataset, batch_size=BATCH_SIZE,\n#                          drop_last=False, shuffle=False, num_workers=2)\n\n# for model_index in range(NUM_MODELS):            \n#     model_path = f\"../input/commonlitmodels13/models1/model_{model_index + 1}.pth\"\n#     print(f\"\\nUsing {model_path}\")\n                        \n#     model = LitModel()\n#     model.load_state_dict(torch.load(model_path, map_location=DEVICE))    \n#     model.to(DEVICE)\n        \n#     all_predictions[model_index] = predict(model, test_loader)\n            \n#     del model\n#     gc.collect()        \n\n# # model1_predictions = all_predictions.mean(axis=0)\n# model1_predictions = np.median(all_predictions,axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:29:32.186683Z","iopub.execute_input":"2021-08-07T10:29:32.187098Z","iopub.status.idle":"2021-08-07T10:29:32.194376Z","shell.execute_reply.started":"2021-08-07T10:29:32.187012Z","shell.execute_reply":"2021-08-07T10:29:32.193512Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [-0.41689962, -0.62006736, -0.36756319, -2.50792193, -1.68248856, -1.48321199,  0.1302613 ]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:29:32.195856Z","iopub.execute_input":"2021-08-07T10:29:32.196471Z","iopub.status.idle":"2021-08-07T10:29:32.205134Z","shell.execute_reply.started":"2021-08-07T10:29:32.196432Z","shell.execute_reply":"2021-08-07T10:29:32.20411Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 2","metadata":{}},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 3","metadata":{}},{"cell_type":"code","source":"# import os\n# import numpy as np\n# import pandas as pd\n# import random\n\n# from transformers import AutoConfig, AutoModel, AutoTokenizer, AdamW, get_linear_schedule_with_warmup, logging\n\n# import torch\n# import torch.nn as nn\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, TensorDataset, SequentialSampler, RandomSampler, DataLoader\n\n# from tqdm.notebook import tqdm\n\n# import gc; gc.enable()\n# from IPython.display import clear_output\n\n# from sklearn.model_selection import StratifiedKFold\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n# INPUT_DIR = '../input/commonlitreadabilityprize'\n# MODEL_DIR = '../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-large'\n# CHECKPOINT_DIR = '../input/commonlitmodels13/models3/'\n\n# DEVICE = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n\n# MAX_LENGTH = 300\n# TEST_BATCH_SIZE = 1\n# HIDDEN_SIZE = 1024\n\n# NUM_FOLDS = 5\n# SEEDS = [113, 71]\n\n# class MeanPoolingModel(nn.Module):\n    \n#     def __init__(self, model_name):\n#         super().__init__()\n        \n#         config = AutoConfig.from_pretrained(model_name)\n#         self.model = AutoModel.from_pretrained(model_name, config=config)\n#         self.linear = nn.Linear(HIDDEN_SIZE, 1)\n#         self.loss = nn.MSELoss()\n        \n#     def forward(self, input_ids, attention_mask, labels=None):\n        \n#         outputs = self.model(input_ids, attention_mask)\n#         last_hidden_state = outputs[0]\n#         input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n#         sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n#         sum_mask = input_mask_expanded.sum(1)\n#         sum_mask = torch.clamp(sum_mask, min=1e-9)\n#         mean_embeddings = sum_embeddings / sum_mask\n#         logits = self.linear(mean_embeddings)\n        \n#         preds = logits.squeeze(-1).squeeze(-1)\n        \n#         if labels is not None:\n#             loss = self.loss(preds.view(-1).float(), labels.view(-1).float())\n#             return loss\n#         else:\n#             return preds\n\n# def get_test_loader(data):\n\n#     x_test = data.excerpt.tolist()\n    \n#     tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)\n\n#     encoded_test = tokenizer.batch_encode_plus(\n#         x_test, \n#         add_special_tokens=True, \n#         return_attention_mask=True, \n#         padding='max_length', \n#         truncation=True,\n#         max_length=MAX_LENGTH, \n#         return_tensors='pt'\n#     )\n\n#     dataset_test = TensorDataset(\n#         encoded_test['input_ids'],\n#         encoded_test['attention_mask']\n#     )\n\n#     dataloader_test = DataLoader(\n#         dataset_test,\n#         sampler = SequentialSampler(dataset_test),\n#         batch_size=TEST_BATCH_SIZE\n#     )\n    \n#     return dataloader_test\n\n# test = test_df\n# test_dataloader = get_test_loader(test)\n# all_predictions = []\n# for seed in SEEDS:\n    \n#     fold_predictions = []\n    \n#     for fold in tqdm(range(NUM_FOLDS)):\n#         model_path = f\"model_{seed + 1}_{fold + 1}.pth\"\n        \n#         print(f\"\\nUsing {model_path}\")\n        \n#         model_path = CHECKPOINT_DIR + f\"model_{seed + 1}_{fold + 1}.pth\"\n#         model = MeanPoolingModel(MODEL_DIR)\n# #         model.load_state_dict(torch.load(model_path)) \n#         model.load_state_dict(torch.load(model_path, map_location=DEVICE)) \n#         model.to(DEVICE)\n#         model.eval()\n\n#         predictions = []\n#         for batch in test_dataloader:\n\n#             batch = tuple(b.to(DEVICE) for b in batch)\n\n#             inputs = {'input_ids':      batch[0],\n#                       'attention_mask': batch[1],\n#                       'labels':         None,\n#                      }\n\n     \n#             preds = model(**inputs).item()\n#             predictions.append(preds)\n            \n#         del model \n#         gc.collect()\n            \n#         fold_predictions.append(predictions)\n#     all_predictions.extend(fold_predictions)\n    \n# # model3_predictions = np.mean(all_predictions,axis=0)\n# model3_predictions = np.median(all_predictions,axis=0)\n","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:29:32.20677Z","iopub.execute_input":"2021-08-07T10:29:32.207162Z","iopub.status.idle":"2021-08-07T10:29:32.214754Z","shell.execute_reply.started":"2021-08-07T10:29:32.207125Z","shell.execute_reply":"2021-08-07T10:29:32.213738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [-0.40176001, -0.37179838, -0.38157192, -2.27679205, -1.86705327, -1.21751082,  0.1195216 ]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:29:32.216708Z","iopub.execute_input":"2021-08-07T10:29:32.217116Z","iopub.status.idle":"2021-08-07T10:29:32.224776Z","shell.execute_reply.started":"2021-08-07T10:29:32.217076Z","shell.execute_reply":"2021-08-07T10:29:32.223854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 4","metadata":{}},{"cell_type":"code","source":"# import os\n# import gc\n# import sys\n# import math\n# import time\n# import tqdm\n# import random\n# import numpy as np\n# import pandas as pd\n# import seaborn as sns\n# from tqdm import tqdm\n# import matplotlib.pyplot as plt\n# from sklearn.model_selection import KFold\n\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# from sklearn.metrics import mean_squared_error\n# from sklearn.model_selection import StratifiedKFold\n\n# import torch\n# import torchvision\n# import torch.nn as nn\n# import torch.optim as optim\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n# from torch.utils.data import RandomSampler, SequentialSampler, Sampler\n# from torch.nn.functional import mse_loss\n# from transformers import AutoModel,AutoTokenizer,get_cosine_schedule_with_warmup, AutoConfig, AdamW\n\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n# plt.style.use('seaborn-talk')\n# # print(plt.style.available)\n# from time import time\n\n# DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# from torch.utils.data import Dataset\n# import torch\n\n# def convert_examples_to_features(text, tokenizer, max_len):\n\n#     tok = tokenizer.encode_plus(\n#         text, \n#         max_length=max_len, \n#         truncation=True,\n#         padding='max_length',\n#     )\n#     return tok\n\n\n# class CLRPDataset(Dataset):\n#     def __init__(self, data, tokenizer, max_len, is_test=False):\n#         self.data = data\n#         self.excerpts = self.data.excerpt.tolist()\n#         if not is_test:\n#             self.targets = self.data.target.tolist()\n            \n#         self.tokenizer = tokenizer\n#         self.is_test = is_test\n#         self.max_len = max_len\n    \n#     def __len__(self):\n#         return len(self.data)\n    \n#     def __getitem__(self, item):\n#         if not self.is_test:\n#             excerpt = self.excerpts[item]\n#             label = self.targets[item]\n#             features = convert_examples_to_features(\n#                 excerpt, self.tokenizer, self.max_len\n#             )\n#             return {\n#                 'input_ids':torch.tensor(features['input_ids'], dtype=torch.long),\n#                 'attention_mask':torch.tensor(features['attention_mask'], dtype=torch.long),\n#                 'label':torch.tensor(label, dtype=torch.float),\n#             }\n#         else:\n#             excerpt = self.excerpts[item]\n#             features = convert_examples_to_features(\n#                 excerpt, self.tokenizer, self.max_len\n#             )\n#             return {\n#                 'input_ids':torch.tensor(features['input_ids'], dtype=torch.long),\n#                 'attention_mask':torch.tensor(features['attention_mask'], dtype=torch.long),\n#             }\n        \n# import torch\n# import torch.nn as nn\n\n# class AttentionHead(nn.Module):\n#     def __init__(self, h_size, hidden_dim=512):\n#         super().__init__()\n#         self.W = nn.Linear(h_size, hidden_dim)\n#         self.V = nn.Linear(hidden_dim, 1)\n        \n#     def forward(self, features):\n#         att = torch.tanh(self.W(features))\n#         score = self.V(att)\n#         attention_weights = torch.softmax(score, dim=1)\n#         context_vector = attention_weights * features\n#         context_vector = torch.sum(context_vector, dim=1)\n\n#         return context_vector\n\n# class CLRPModel(nn.Module):\n#     def __init__(self,transformer,config):\n#         super(CLRPModel,self).__init__()\n#         self.h_size = config.hidden_size\n#         self.transformer = transformer\n#         self.head = AttentionHead(self.h_size*4)\n#         self.linear = nn.Linear(self.h_size*8, self.h_size // 2)\n#         self.linear_out = nn.Linear(self.h_size // 2, 1)\n#         self.tanh = nn.Tanh()\n              \n#     def forward(self, input_ids, attention_mask):\n#         transformer_out = self.transformer(input_ids, attention_mask)\n       \n#         all_hidden_states = torch.stack(transformer_out.hidden_states)\n#         cat_over_last_layers = torch.cat(\n#             (all_hidden_states[-1], all_hidden_states[-2], all_hidden_states[-3], all_hidden_states[-4]),-1\n#         )\n        \n#         cls_pooling = cat_over_last_layers[:, 0]   \n#         head_logits = self.head(cat_over_last_layers)\n#         logits = self.tanh(self.linear(torch.cat([head_logits, cls_pooling], -1)))\n#         y_hat = self.linear_out(logits)\n        \n#         return y_hat\n\n# class Config:\n#     model_name = 'roberta-large'\n#     output_hidden_states = True\n#     epochs = 5\n# #     evaluate_interval = 40\n#     batch_size = 8\n#     device = 'cuda'\n#     seed = 42\n#     max_len = 256\n#     lr = 1e-5\n#     wd = 0.01\n# #     eval_schedule = [(float('inf'), 40), (0.5, 30), (0.49, 20), (0.48, 10), (0.47, 3), (0, 0)]\n#     eval_schedule = [(float('inf'), 40), (0.47, 20), (0.46, 10), (0, 0)]\n\n#     gradient_accumulation = 2\n\n# test_df = pd.read_csv(\"/kaggle/input/commonlitreadabilityprize/test.csv\")\n# # tokenizer = torch.load('../input/commonlitmodels4/models4/roberta-tokenizer.pt')\n# tokenizer = AutoTokenizer.from_pretrained(\"../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-large\")\n# models_preds = []\n# n_models = 5\n\n# for model_num in range(n_models):\n#     print(f'Inference#{model_num+1}/{n_models}')\n#     test_ds = CLRPDataset(data=test_df, tokenizer=tokenizer, max_len=Config.max_len, is_test=True)\n#     test_sampler = SequentialSampler(test_ds)\n#     test_dataloader = DataLoader(test_ds, sampler = test_sampler, batch_size=Config.batch_size)\n#     model = torch.load(f'../input/commonlitmodels4/models4/best_model_{model_num}.pt', map_location=DEVICE)\n#     all_preds = []\n#     model.eval()\n\n#     for step,batch in enumerate(test_dataloader):\n# #         sent_id, mask = batch['input_ids'].to(Config.device), batch['attention_mask'].to(Config.device)\n#         sent_id, mask = batch['input_ids'].to(DEVICE), batch['attention_mask'].to(DEVICE)\n#         with torch.no_grad():\n#             preds = model(sent_id, mask)\n#             all_preds += preds.flatten().cpu().tolist()\n    \n#     models_preds.append(all_preds)\n#     del model\n#     gc.collect()\n    \n# models_preds = np.array(models_preds)\n# # print(models_preds.shape)\n# # print(models_preds)\n# # all_preds_4 = models_preds.mean(axis=0)\n# all_preds_4 = np.median(models_preds,axis=0)\n# # print(all_preds_4.shape)\n\n# # [-0.37045592, -0.41978189, -0.44240755, -2.26042914, -1.79838932, -1.28246176,  0.26374587]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:29:32.227068Z","iopub.execute_input":"2021-08-07T10:29:32.227609Z","iopub.status.idle":"2021-08-07T10:29:32.235926Z","shell.execute_reply.started":"2021-08-07T10:29:32.227517Z","shell.execute_reply":"2021-08-07T10:29:32.234703Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import gc\n# import sys\n# import math\n# import time\n# import tqdm\n# import random\n# import numpy as np\n# import pandas as pd\n# import seaborn as sns\n# from tqdm import tqdm\n# import matplotlib.pyplot as plt\n# from sklearn.model_selection import KFold\n\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# from sklearn.metrics import mean_squared_error\n# from sklearn.model_selection import StratifiedKFold\n\n# import torch\n# import torchvision\n# import torch.nn as nn\n# import torch.optim as optim\n# import torch.nn.functional as F\n# from torch.utils.data import Dataset, DataLoader\n# from torch.utils.data import RandomSampler, SequentialSampler, Sampler\n# from torch.nn.functional import mse_loss\n# from transformers import AutoModel,AutoTokenizer,get_cosine_schedule_with_warmup, AutoConfig, AdamW\n\n# import matplotlib.pyplot as plt\n# import seaborn as sns\n# plt.style.use('seaborn-talk')\n# # print(plt.style.available)\n# from time import time\n\n# DEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\n# from torch.utils.data import Dataset\n# import torch\n\n# def convert_examples_to_features(text, tokenizer, max_len):\n\n#     tok = tokenizer.encode_plus(\n#         text, \n#         max_length=max_len, \n#         truncation=True,\n#         padding='max_length',\n#     )\n#     return tok\n\n\n# class CLRPDataset(Dataset):\n#     def __init__(self, data, tokenizer, max_len, is_test=False):\n#         self.data = data\n#         self.excerpts = self.data.excerpt.tolist()\n#         if not is_test:\n#             self.targets = self.data.target.tolist()\n            \n#         self.tokenizer = tokenizer\n#         self.is_test = is_test\n#         self.max_len = max_len\n    \n#     def __len__(self):\n#         return len(self.data)\n    \n#     def __getitem__(self, item):\n#         if not self.is_test:\n#             excerpt = self.excerpts[item]\n#             label = self.targets[item]\n#             features = convert_examples_to_features(\n#                 excerpt, self.tokenizer, self.max_len\n#             )\n#             return {\n#                 'input_ids':torch.tensor(features['input_ids'], dtype=torch.long),\n#                 'attention_mask':torch.tensor(features['attention_mask'], dtype=torch.long),\n#                 'label':torch.tensor(label, dtype=torch.float),\n#             }\n#         else:\n#             excerpt = self.excerpts[item]\n#             features = convert_examples_to_features(\n#                 excerpt, self.tokenizer, self.max_len\n#             )\n#             return {\n#                 'input_ids':torch.tensor(features['input_ids'], dtype=torch.long),\n#                 'attention_mask':torch.tensor(features['attention_mask'], dtype=torch.long),\n#             }\n        \n        \n# import torch\n# import torch.nn as nn\n\n# class AttentionHead(nn.Module):\n#     def __init__(self, h_size, hidden_dim=512):\n#         super().__init__()\n#         self.W = nn.Linear(h_size, hidden_dim)\n#         self.V = nn.Linear(hidden_dim, 1)\n        \n#     def forward(self, features):\n#         att = torch.tanh(self.W(features))\n#         score = self.V(att)\n#         attention_weights = torch.softmax(score, dim=1)\n#         context_vector = attention_weights * features\n#         context_vector = torch.sum(context_vector, dim=1)\n\n#         return context_vector\n\n# class CLRPModel(nn.Module):\n#     def __init__(self,transformer,config):\n#         super(CLRPModel,self).__init__()\n#         self.h_size = config.hidden_size\n#         self.transformer = transformer\n#         self.head = AttentionHead(self.h_size*4)\n#         self.linear = nn.Linear(self.h_size*2, 1)\n#         self.linear_out = nn.Linear(self.h_size*8, 1)\n\n              \n#     def forward(self, input_ids, attention_mask):\n#         transformer_out = self.transformer(input_ids, attention_mask)\n       \n#         all_hidden_states = torch.stack(transformer_out.hidden_states)\n#         cat_over_last_layers = torch.cat(\n#             (all_hidden_states[-1], all_hidden_states[-2], all_hidden_states[-3], all_hidden_states[-4]),-1\n#         )\n        \n#         cls_pooling = cat_over_last_layers[:, 0]   \n#         head_logits = self.head(cat_over_last_layers)\n#         y_hat = self.linear_out(torch.cat([head_logits, cls_pooling], -1))\n        \n#         return y_hat\n\n# class Config:\n#     model_name = 'roberta-large'\n#     output_hidden_states = True\n#     epochs = 5\n# #     evaluate_interval = 40\n#     batch_size = 8\n#     device = 'cuda'\n#     seed = 42\n#     max_len = 256\n#     lr = 1e-5\n#     wd = 0.01\n# #     eval_schedule = [(float('inf'), 40), (0.5, 30), (0.49, 20), (0.48, 10), (0.47, 3), (0, 0)]\n#     eval_schedule = [(float('inf'), 40), (0.47, 20), (0.46, 10), (0, 0)]\n\n#     gradient_accumulation = 2\n    \n# # tokenizer = torch.load('../input/commonlitmodels4/models4/roberta-tokenizer.pt')\n# tokenizer = AutoTokenizer.from_pretrained(\"../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-large\")\n# models_preds = []\n# n_models = 5\n\n# for model_num in range(n_models):\n#     print(f'Inference#{model_num+1}/{n_models}')\n#     test_ds = CLRPDataset(data=test_df, tokenizer=tokenizer, max_len=Config.max_len, is_test=True)\n#     test_sampler = SequentialSampler(test_ds)\n#     test_dataloader = DataLoader(test_ds, sampler = test_sampler, batch_size=Config.batch_size)\n    \n#     config = AutoConfig.from_pretrained(\"../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-large\")\n#     config.update({\n#             \"hidden_dropout_prob\": 0.0,\n#             \"layer_norm_eps\": 1e-7,\n#             \"output_hidden_states\": True\n#             }) \n#     transformer = AutoModel.from_pretrained(\"../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-large\", config=config)\n    \n#     model = CLRPModel(transformer, config)\n# #     model.load_state_dict(torch.load(f'../input/clrp-roberta-large-0459/clrp_roberta_large_0459/best_model_{model_num}.bin', map_location=DEVICE))\n#     model.load_state_dict(torch.load(f'../input/clrp-model4-large-0465/best_model_{model_num}.bin', map_location=DEVICE))\n#     model.to(DEVICE)\n#     model.eval()\n#     all_preds = []\n    \n#     for step,batch in enumerate(test_dataloader):\n# #         sent_id, mask = batch['input_ids'].to(Config.device), batch['attention_mask'].to(Config.device)\n#         sent_id, mask = batch['input_ids'].to(DEVICE), batch['attention_mask'].to(DEVICE)\n#         with torch.no_grad():\n#             preds = model(sent_id, mask)\n#             all_preds += preds.flatten().cpu().tolist()\n    \n#     models_preds.append(all_preds)\n#     del model\n#     gc.collect()\n    \n# models_preds = np.array(models_preds)\n# # print(models_preds.shape)\n# # print(models_preds)\n# # all_preds_4 = models_preds.mean(axis=0)\n# all_preds_4 = np.median(models_preds,axis=0)\n# # print(all_preds_4.shape)\n# # [-0.39522862, -0.38330543, -0.37382132, -2.38678098, -1.8379159 , -1.23651814,  0.18913254]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:29:32.237737Z","iopub.execute_input":"2021-08-07T10:29:32.238141Z","iopub.status.idle":"2021-08-07T10:29:32.248033Z","shell.execute_reply.started":"2021-08-07T10:29:32.238104Z","shell.execute_reply":"2021-08-07T10:29:32.247208Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 5","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport sys\nimport math\nimport time\nimport tqdm\nimport random\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom sklearn.model_selection import KFold\n\nimport warnings\nwarnings.filterwarnings('ignore')\n\nfrom sklearn.metrics import mean_squared_error\nfrom sklearn.model_selection import StratifiedKFold\n\nimport torch\nimport torchvision\nimport torch.nn as nn\nimport torch.optim as optim\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data import RandomSampler, SequentialSampler, Sampler\nfrom torch.nn.functional import mse_loss\nfrom transformers import AutoModel,AutoTokenizer,get_cosine_schedule_with_warmup, AutoConfig, AdamW\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nplt.style.use('seaborn-talk')\n# print(plt.style.available)\nfrom time import time\n\nDEVICE = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n\nfrom torch.utils.data import Dataset\nimport torch\n\ndef convert_examples_to_features(text, tokenizer, max_len):\n\n    tok = tokenizer.encode_plus(\n        text, \n        max_length=max_len, \n        truncation=True,\n        padding='max_length',\n    )\n    return tok\n\n\nclass CLRPDataset(Dataset):\n    def __init__(self, data, tokenizer, max_len, is_test=False):\n        self.data = data\n        self.excerpts = self.data.excerpt.tolist()\n        if not is_test:\n            self.targets = self.data.target.tolist()\n            \n        self.tokenizer = tokenizer\n        self.is_test = is_test\n        self.max_len = max_len\n    \n    def __len__(self):\n        return len(self.data)\n    \n    def __getitem__(self, item):\n        if not self.is_test:\n            excerpt = self.excerpts[item]\n            label = self.targets[item]\n            features = convert_examples_to_features(\n                excerpt, self.tokenizer, self.max_len\n            )\n            return {\n                'input_ids':torch.tensor(features['input_ids'], dtype=torch.long),\n                'attention_mask':torch.tensor(features['attention_mask'], dtype=torch.long),\n                'label':torch.tensor(label, dtype=torch.float),\n            }\n        else:\n            excerpt = self.excerpts[item]\n            features = convert_examples_to_features(\n                excerpt, self.tokenizer, self.max_len\n            )\n            return {\n                'input_ids':torch.tensor(features['input_ids'], dtype=torch.long),\n                'attention_mask':torch.tensor(features['attention_mask'], dtype=torch.long),\n            }\n        \n        \nimport torch\nimport torch.nn as nn\n\nclass AttentionHead(nn.Module):\n    def __init__(self, h_size, hidden_dim=512):\n        super().__init__()\n        self.W = nn.Linear(h_size, hidden_dim)\n        self.V = nn.Linear(hidden_dim, 1)\n        \n    def forward(self, features):\n        att = torch.tanh(self.W(features))\n        score = self.V(att)\n        attention_weights = torch.softmax(score, dim=1)\n        context_vector = attention_weights * features\n        context_vector = torch.sum(context_vector, dim=1)\n\n        return context_vector\n\nclass CLRPModel(nn.Module):\n    def __init__(self,transformer,config):\n        super(CLRPModel,self).__init__()\n        self.h_size = config.hidden_size\n        self.transformer = transformer\n        self.head = AttentionHead(self.h_size*4)\n        self.linear = nn.Linear(self.h_size*2, 1)\n        self.linear_out = nn.Linear(self.h_size*8, 1)\n\n              \n    def forward(self, input_ids, attention_mask):\n        transformer_out = self.transformer(input_ids, attention_mask)\n       \n        all_hidden_states = torch.stack(transformer_out.hidden_states)\n        cat_over_last_layers = torch.cat(\n            (all_hidden_states[-1], all_hidden_states[-2], all_hidden_states[-3], all_hidden_states[-4]),-1\n        )\n        \n        cls_pooling = cat_over_last_layers[:, 0]   \n        head_logits = self.head(cat_over_last_layers)\n        y_hat = self.linear_out(torch.cat([head_logits, cls_pooling], -1))\n        \n        return y_hat\n\nclass Config:\n    model_name = 'roberta-large'\n    output_hidden_states = True\n    epochs = 5\n#     evaluate_interval = 40\n    batch_size = 8\n    device = 'cuda'\n    seed = 42\n    max_len = 256\n    lr = 1e-5\n    wd = 0.01\n#     eval_schedule = [(float('inf'), 40), (0.5, 30), (0.49, 20), (0.48, 10), (0.47, 3), (0, 0)]\n    eval_schedule = [(float('inf'), 40), (0.47, 20), (0.46, 10), (0, 0)]\n\n    gradient_accumulation = 2\n    \n# tokenizer = torch.load('../input/commonlitmodels4/models4/roberta-tokenizer.pt')\ntokenizer = AutoTokenizer.from_pretrained(\"../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-large\")\nmodels_preds = []\nn_models = 5\n\nfor model_num in range(n_models):\n    print(f'Inference#{model_num+1}/{n_models}')\n    test_ds = CLRPDataset(data=test_df, tokenizer=tokenizer, max_len=Config.max_len, is_test=True)\n    test_sampler = SequentialSampler(test_ds)\n    test_dataloader = DataLoader(test_ds, sampler = test_sampler, batch_size=Config.batch_size)\n    \n    config = AutoConfig.from_pretrained(\"../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-large\")\n    config.update({\n            \"hidden_dropout_prob\": 0.0,\n            \"layer_norm_eps\": 1e-7,\n            \"output_hidden_states\": True\n            }) \n    transformer = AutoModel.from_pretrained(\"../input/roberta-transformers-pytorch/RoBERTa_Transformers_Pytorch/roberta-large\", config=config)\n    \n    model = CLRPModel(transformer, config)\n#     model.load_state_dict(torch.load(f'../input/clrp-roberta-large-0459/clrp_roberta_large_0459/best_model_{model_num}.bin', map_location=DEVICE))\n    model.load_state_dict(torch.load(f'../input/clrp-model4-large-0465/best_model_{model_num}.bin', map_location=DEVICE))\n    model.to(DEVICE)\n    model.eval()\n    all_preds = []\n    \n    for step,batch in enumerate(test_dataloader):\n#         sent_id, mask = batch['input_ids'].to(Config.device), batch['attention_mask'].to(Config.device)\n        sent_id, mask = batch['input_ids'].to(DEVICE), batch['attention_mask'].to(DEVICE)\n        with torch.no_grad():\n            preds = model(sent_id, mask)\n            all_preds += preds.flatten().cpu().tolist()\n    \n    models_preds.append(all_preds)\n    del model\n    gc.collect()\n    \nmodels_preds = np.array(models_preds)\n# print(models_preds.shape)\n# print(models_preds)\nall_preds_5 = models_preds.mean(axis=0)\n# all_preds_5 = np.median(models_preds,axis=0)\n# print(all_preds_5.shape)\n# [-0.39522862, -0.38330543, -0.37382132, -2.38678098, -1.8379159 , -1.23651814,  0.18913254]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:29:32.28761Z","iopub.execute_input":"2021-08-07T10:29:32.287902Z","iopub.status.idle":"2021-08-07T10:31:46.275847Z","shell.execute_reply.started":"2021-08-07T10:29:32.287859Z","shell.execute_reply":"2021-08-07T10:31:46.274981Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission_df.target = all_preds_5\ntrain_target_mean = train_df.target.mean()\ntest_target_mean = submission_df.target.mean()\n\nsubmission_df.target = submission_df.target - (test_target_mean-train_target_mean)\nsubmission_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:35:22.835124Z","iopub.execute_input":"2021-08-07T10:35:22.835463Z","iopub.status.idle":"2021-08-07T10:35:22.846998Z","shell.execute_reply.started":"2021-08-07T10:35:22.835428Z","shell.execute_reply":"2021-08-07T10:35:22.846182Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model 6","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import gc\n# import sys\n# import cv2\n# import math\n# import time\n# import tqdm\n# import random\n# import numpy as np\n# import pandas as pd\n# import seaborn as sns\n# from tqdm import tqdm\n# import matplotlib.pyplot as plt\n\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# from sklearn.svm import SVR\n\n# from sklearn.metrics import mean_squared_error\n# from sklearn.model_selection import KFold,StratifiedKFold\n\n# import torch\n# import torchvision\n# import torch.nn as nn\n# import torch.optim as optim\n# import torch.nn.functional as F\n# from torch.optim import Adam, lr_scheduler\n# from torch.utils.data import Dataset, DataLoader\n\n# from transformers import (AutoModel, AutoTokenizer, \n#                           AutoModelForSequenceClassification)\n\n# train_data = pd.read_csv('../input/commonlitreadabilityprize/train.csv')\n# test_data = pd.read_csv('../input/commonlitreadabilityprize/test.csv')\n# sample = pd.read_csv('../input/commonlitreadabilityprize/sample_submission.csv')\n\n# #対数分だけnum_binsに分割\n# #ビニング処理,データをbin数でグループ分けする\n# num_bins = int(np.floor(1 + np.log2(len(train_data))))\n# train_data.loc[:,'bins'] = pd.cut(train_data['target'],bins=num_bins,labels=False)\n\n# target = train_data['target'].to_numpy()\n# bins = train_data.bins.to_numpy()\n\n# #平均二条誤差\n# def rmse_score(y_true,y_pred):\n#     return np.sqrt(mean_squared_error(y_true,y_pred))\n# config = {\n#     'batch_size':128,\n#     'max_len':256,\n#     'nfolds':5,\n#     'seed':42,\n# }\n\n# def seed_everything(seed=42):\n#     random.seed(seed)\n#     os.environ['PYTHONASSEED'] = str(seed)\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n#     torch.backends.cudnn.benchmark = True\n\n# seed_everything(seed=config['seed'])\n# class CLRPDataset(Dataset):\n#     def __init__(self,df,tokenizer):\n#         self.excerpt = df['excerpt'].to_numpy()\n#         self.tokenizer = tokenizer\n    \n#     def __getitem__(self,idx):\n#         encode = self.tokenizer(self.excerpt[idx],return_tensors='pt',\n#                                 max_length=config['max_len'],\n#                                 padding='max_length',truncation=True)\n#         return encode\n    \n#     def __len__(self):\n#         return len(self.excerpt)\n# class AttentionHead(nn.Module):\n#     def __init__(self, in_features, hidden_dim, num_targets):\n#         super().__init__()\n#         self.in_features = in_features\n#         self.middle_features = hidden_dim\n\n#         self.W = nn.Linear(in_features, hidden_dim)\n#         self.V = nn.Linear(hidden_dim, 1)\n#         self.out_features = hidden_dim\n\n#     def forward(self, features):\n#         att = torch.tanh(self.W(features))\n\n#         score = self.V(att)\n\n#         attention_weights = torch.softmax(score, dim=1)\n\n#         context_vector = attention_weights * features\n#         context_vector = torch.sum(context_vector, dim=1)\n\n#         return context_vector\n# MODEL = \"../input/clrp-pytorch-roberta-pretrain-robertalarge/clrp_roberta_large\"\n# class Model(nn.Module):\n#     def __init__(self):\n#         super(Model,self).__init__()\n#         self.roberta = AutoModel.from_pretrained(MODEL)    \n#         #changed attentionHead Dimension from 768 to 1024 by changing model from roberta-base to roberta-large\n#         self.head = AttentionHead(1024,1024,1)\n#         self.dropout = nn.Dropout(0.1)\n#         self.linear = nn.Linear(self.head.out_features,1)\n\n#     def forward(self,**xb):\n#         x = self.roberta(**xb)[0]\n#         x = self.head(x)\n#         return x\n# #ここが新規\n# def get_embeddings(df,path,plot_losses=True, verbose=True):\n#     #cuda使えたら使う構文\n#     device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n#     print(f\"{device} is used\")\n            \n#     model = Model()\n#     model.load_state_dict(torch.load(path))\n#     model.to(device)\n#     model.eval()\n    \n#     tokenizer = AutoTokenizer.from_pretrained(MODEL)\n    \n#     ds = CLRPDataset(df,tokenizer)\n#     dl = DataLoader(ds,\n#                   batch_size = config[\"batch_size\"],\n#                   shuffle=False,\n#                   num_workers = 4,\n#                   pin_memory=True,\n#                   drop_last=False\n#                  )\n        \n#     #以下でpredictionsを抽出するために使った構文を使ってembeddingsをreturnしている.\n#     #SVMの手法とは、embeddingsの意味は？\n#     embeddings = list()\n#     with torch.no_grad():\n#         for i, inputs in tqdm(enumerate(dl)):\n#             inputs = {key:val.reshape(val.shape[0],-1).to(device) for key,val in inputs.items()}\n#             outputs = model(**inputs)\n#             outputs = outputs.detach().cpu().numpy()\n#             embeddings.extend(outputs)\n#     return np.array(embeddings)\n \n# #train/testでembeddingsを取得している\n# train_embeddings1 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model0/model0.bin')\n# test_embeddings1 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model0/model0.bin')\n\n# train_embeddings2 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model1/model1.bin')\n# test_embeddings2 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model1/model1.bin')\n\n# train_embeddings3 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model2/model2.bin')\n# test_embeddings3 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model2/model2.bin')\n\n# train_embeddings4 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model3/model3.bin')\n# test_embeddings4 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model3/model3.bin')\n\n# train_embeddings5 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model4/model4.bin')\n# test_embeddings5 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model4/model4.bin')\n\n# #SVMをアンサンブル処理している\n# def get_preds_svm(X,y,X_test,bins=bins,nfolds=10,C=10,kernel='rbf'):\n#     scores = list()\n#     preds = np.zeros((X_test.shape[0]))\n    \n#     kfold = StratifiedKFold(n_splits=config['nfolds'],shuffle=True,random_state=config['seed'])\n#     for k, (train_idx,valid_idx) in enumerate(kfold.split(X,bins)):\n#         model = SVR(C=C,kernel=kernel,gamma='auto')\n#         X_train,y_train = X[train_idx], y[train_idx]\n#         X_valid,y_valid = X[valid_idx], y[valid_idx]\n        \n#         model.fit(X_train,y_train)\n#         prediction = model.predict(X_valid)\n#         score = rmse_score(prediction,y_valid)\n#         print(f'Fold {k} , rmse score: {score}')\n#         scores.append(score)\n#         preds += model.predict(X_test)\n#         del model\n#         gc.collect()\n        \n#     print(\"mean rmse\",np.mean(scores))\n#     return np.array(preds)/nfolds\n# svm_preds1 = get_preds_svm(train_embeddings1,target,test_embeddings1)\n# svm_preds2 = get_preds_svm(train_embeddings2,target,test_embeddings2)\n# svm_preds3 = get_preds_svm(train_embeddings3,target,test_embeddings3)\n# svm_preds4 = get_preds_svm(train_embeddings4,target,test_embeddings4)\n# svm_preds5 = get_preds_svm(train_embeddings5,target,test_embeddings5)\n\n# model6_predictions = np.mean([svm_preds1,svm_preds2,svm_preds3,svm_preds4,svm_preds5],axis=0)\n# # model6_predictions = np.median([svm_preds1,svm_preds2,svm_preds3,svm_preds4,svm_preds5],axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:31:46.534266Z","iopub.execute_input":"2021-08-07T10:31:46.534553Z","iopub.status.idle":"2021-08-07T10:31:46.544973Z","shell.execute_reply.started":"2021-08-07T10:31:46.534518Z","shell.execute_reply":"2021-08-07T10:31:46.544009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import gc\n# import sys\n# import cv2\n# import math\n# import time\n# import tqdm\n# import random\n# import numpy as np\n# import pandas as pd\n# import seaborn as sns\n# from tqdm import tqdm\n# import matplotlib.pyplot as plt\n\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# from sklearn.svm import SVR\n# from sklearn.ensemble import RandomForestRegressor\n# from lightgbm import LGBMRegressor\n# from sklearn.metrics import mean_squared_error\n# from sklearn.model_selection import KFold,StratifiedKFold\n\n# import torch\n# import torchvision\n# import torch.nn as nn\n# import torch.optim as optim\n# import torch.nn.functional as F\n# from torch.optim import Adam, lr_scheduler\n# from torch.utils.data import Dataset, DataLoader\n\n# from transformers import (AutoModel, AutoTokenizer, \n#                           AutoModelForSequenceClassification)\n\n# import plotly.express as px\n# import plotly.graph_objs as go\n# import plotly.figure_factory as ff\n\n\n# from colorama import Fore, Back, Style\n# y_ = Fore.YELLOW\n# r_ = Fore.RED\n# g_ = Fore.GREEN\n# b_ = Fore.BLUE\n# m_ = Fore.MAGENTA\n# c_ = Fore.CYAN\n# sr_ = Style.RESET_ALL\n# train_data = pd.read_csv('../input/commonlitreadabilityprize/train.csv')\n# test_data = pd.read_csv('../input/commonlitreadabilityprize/test.csv')\n# sample = pd.read_csv('../input/commonlitreadabilityprize/sample_submission.csv')\n\n# #対数分だけnum_binsに分割\n# #ビニング処理,データをbin数でグループ分けする\n# num_bins = int(np.floor(1 + np.log2(len(train_data))))\n# train_data.loc[:,'bins'] = pd.cut(train_data['target'],bins=num_bins,labels=False)\n\n# target = train_data['target'].to_numpy()\n# bins = train_data.bins.to_numpy()\n\n# #平均二条誤差\n# def rmse_score(y_true,y_pred):\n#     return np.sqrt(mean_squared_error(y_true,y_pred))\n# config = {\n#     'batch_size':128,\n#     'max_len':256,\n#     'nfolds':10,\n#     'seed':42,\n# }\n\n# def seed_everything(seed=42):\n#     random.seed(seed)\n#     os.environ['PYTHONASSEED'] = str(seed)\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n#     torch.backends.cudnn.benchmark = True\n\n# seed_everything(seed=config['seed'])\n# class CLRPDataset(Dataset):\n#     def __init__(self,df,tokenizer):\n#         self.excerpt = df['excerpt'].to_numpy()\n#         self.tokenizer = tokenizer\n    \n#     def __getitem__(self,idx):\n#         encode = self.tokenizer(self.excerpt[idx],return_tensors='pt',\n#                                 max_length=config['max_len'],\n#                                 padding='max_length',truncation=True)\n#         return encode\n    \n#     def __len__(self):\n#         return len(self.excerpt)\n# class AttentionHead(nn.Module):\n#     def __init__(self, in_features, hidden_dim, num_targets):\n#         super().__init__()\n#         self.in_features = in_features\n#         self.middle_features = hidden_dim\n\n#         self.W = nn.Linear(in_features, hidden_dim)\n#         self.V = nn.Linear(hidden_dim, 1)\n#         self.out_features = hidden_dim\n\n#     def forward(self, features):\n#         att = torch.tanh(self.W(features))\n\n#         score = self.V(att)\n\n#         attention_weights = torch.softmax(score, dim=1)\n\n#         context_vector = attention_weights * features\n#         context_vector = torch.sum(context_vector, dim=1)\n\n#         return context_vector\n# MODEL = \"../input/clrp-pytorch-roberta-pretrain-robertalarge/clrp_roberta_large\"\n# class Model(nn.Module):\n#     def __init__(self):\n#         super(Model,self).__init__()\n#         self.roberta = AutoModel.from_pretrained(MODEL)    \n#         #changed attentionHead Dimension from 768 to 1024 by changing model from roberta-base to roberta-large\n#         self.head = AttentionHead(1024,1024,1)\n#         self.dropout = nn.Dropout(0.1)\n#         self.linear = nn.Linear(self.head.out_features,1)\n\n#     def forward(self,**xb):\n#         x = self.roberta(**xb)[0]\n#         x = self.head(x)\n#         return x\n# #ここが新規\n# def get_embeddings(df,path,plot_losses=True, verbose=True):\n#     #cuda使えたら使う構文\n#     device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n#     print(f\"{device} is used\")\n            \n#     model = Model()\n#     model.load_state_dict(torch.load(path))\n#     model.to(device)\n#     model.eval()\n    \n#     tokenizer = AutoTokenizer.from_pretrained(MODEL)\n    \n#     ds = CLRPDataset(df,tokenizer)\n#     dl = DataLoader(ds,\n#                   batch_size = config[\"batch_size\"],\n#                   shuffle=False,\n#                   num_workers = 4,\n#                   pin_memory=True,\n#                   drop_last=False\n#                  )\n        \n#     #以下でpredictionsを抽出するために使った構文を使ってembeddingsをreturnしている.\n#     #SVMの手法とは、embeddingsの意味は？\n#     embeddings = list()\n#     with torch.no_grad():\n#         for i, inputs in tqdm(enumerate(dl)):\n#             inputs = {key:val.reshape(val.shape[0],-1).to(device) for key,val in inputs.items()}\n#             outputs = model(**inputs)\n#             outputs = outputs.detach().cpu().numpy()\n#             embeddings.extend(outputs)\n#     return np.array(embeddings)\n\n# verbose = 0\n\n# #train/testでembeddingsを取得している\n# train_embeddings1 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model0/model0.bin')\n# test_embeddings1 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model0/model0.bin')\n# # model1 = RandomForestRegressor(verbose=verbose,n_jobs=-1)\n# model1 = LGBMRegressor(n_jobs=-1)\n# model1.fit(train_embeddings1,target)\n# preds1 = model1.predict(test_embeddings1)\n# # print(preds1)\n# del model1,train_embeddings1,test_embeddings1\n# gc.collect()\n\n# train_embeddings2 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model1/model1.bin')\n# test_embeddings2 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model1/model1.bin')\n# model2 = LGBMRegressor(n_jobs=-1)\n# model2.fit(train_embeddings2,target)\n# preds2 = model2.predict(test_embeddings2)\n# # print(preds2)\n# del model2,train_embeddings2,test_embeddings2\n# gc.collect()\n\n# train_embeddings3 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model2/model2.bin')\n# test_embeddings3 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model2/model2.bin')\n# model3 = LGBMRegressor(n_jobs=-1)\n# model3.fit(train_embeddings3,target)\n# preds3 = model3.predict(test_embeddings3)\n# # print(preds3)\n# del model3,train_embeddings3,test_embeddings3\n# gc.collect()\n\n# train_embeddings4 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model3/model3.bin')\n# test_embeddings4 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model3/model3.bin')\n# model4 = LGBMRegressor(n_jobs=-1)\n# model4.fit(train_embeddings4,target)\n# preds4 = model4.predict(test_embeddings4)\n# # print(preds4)\n# del model4,train_embeddings4,test_embeddings4\n# gc.collect()\n\n# train_embeddings5 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model4/model4.bin')\n# test_embeddings5 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model4/model4.bin')\n# model5 = LGBMRegressor(n_jobs=-1)\n# model5.fit(train_embeddings5,target)\n# preds5 = model5.predict(test_embeddings5)\n# # print(preds5)\n# del model5,train_embeddings5,test_embeddings5\n# gc.collect()\n\n# ml_preds = np.mean([preds1,preds2,preds3,preds4,preds5],axis=0)\n# # ml_preds = np.median([preds1,preds2,preds3,preds4,preds5],axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:31:46.546487Z","iopub.execute_input":"2021-08-07T10:31:46.546914Z","iopub.status.idle":"2021-08-07T10:31:46.556569Z","shell.execute_reply.started":"2021-08-07T10:31:46.546875Z","shell.execute_reply":"2021-08-07T10:31:46.555495Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import os\n# import gc\n# import sys\n# import cv2\n# import math\n# import time\n# import tqdm\n# import random\n# import numpy as np\n# import pandas as pd\n# import seaborn as sns\n# from tqdm import tqdm\n# import matplotlib.pyplot as plt\n\n# import warnings\n# warnings.filterwarnings('ignore')\n\n# from sklearn.svm import SVR\n# from sklearn.ensemble import RandomForestRegressor\n# from lightgbm import LGBMRegressor\n# from sklearn.metrics import mean_squared_error\n# from sklearn.model_selection import KFold,StratifiedKFold\n# # import cudf, cuml, cupy\n# # from cuml.svm import SVR\n# # from cuml.ensemble import RandomForestRegressor\n\n# import torch\n# import torchvision\n# import torch.nn as nn\n# import torch.optim as optim\n# import torch.nn.functional as F\n# from torch.optim import Adam, lr_scheduler\n# from torch.utils.data import Dataset, DataLoader\n\n# from transformers import (AutoModel, AutoTokenizer, \n#                           AutoModelForSequenceClassification)\n\n# import plotly.express as px\n# import plotly.graph_objs as go\n# import plotly.figure_factory as ff\n\n\n# from colorama import Fore, Back, Style\n# y_ = Fore.YELLOW\n# r_ = Fore.RED\n# g_ = Fore.GREEN\n# b_ = Fore.BLUE\n# m_ = Fore.MAGENTA\n# c_ = Fore.CYAN\n# sr_ = Style.RESET_ALL\n# train_data = pd.read_csv('../input/commonlitreadabilityprize/train.csv')\n# test_data = pd.read_csv('../input/commonlitreadabilityprize/test.csv')\n# sample = pd.read_csv('../input/commonlitreadabilityprize/sample_submission.csv')\n\n# #対数分だけnum_binsに分割\n# #ビニング処理,データをbin数でグループ分けする\n# num_bins = int(np.floor(1 + np.log2(len(train_data))))\n# train_data.loc[:,'bins'] = pd.cut(train_data['target'],bins=num_bins,labels=False)\n\n# target = train_data['target'].to_numpy()\n# bins = train_data.bins.to_numpy()\n\n# #平均二条誤差\n# def rmse_score(y_true,y_pred):\n#     return np.sqrt(mean_squared_error(y_true,y_pred))\n# config = {\n#     'batch_size':128,\n#     'max_len':256,\n#     'nfolds':10,\n#     'seed':42,\n# }\n\n# def seed_everything(seed=42):\n#     random.seed(seed)\n#     os.environ['PYTHONASSEED'] = str(seed)\n#     np.random.seed(seed)\n#     torch.manual_seed(seed)\n#     torch.cuda.manual_seed(seed)\n#     torch.backends.cudnn.deterministic = True\n#     torch.backends.cudnn.benchmark = True\n\n# seed_everything(seed=config['seed'])\n# class CLRPDataset(Dataset):\n#     def __init__(self,df,tokenizer):\n#         self.excerpt = df['excerpt'].to_numpy()\n#         self.tokenizer = tokenizer\n    \n#     def __getitem__(self,idx):\n#         encode = self.tokenizer(self.excerpt[idx],return_tensors='pt',\n#                                 max_length=config['max_len'],\n#                                 padding='max_length',truncation=True)\n#         return encode\n    \n#     def __len__(self):\n#         return len(self.excerpt)\n# class AttentionHead(nn.Module):\n#     def __init__(self, in_features, hidden_dim, num_targets):\n#         super().__init__()\n#         self.in_features = in_features\n#         self.middle_features = hidden_dim\n\n#         self.W = nn.Linear(in_features, hidden_dim)\n#         self.V = nn.Linear(hidden_dim, 1)\n#         self.out_features = hidden_dim\n\n#     def forward(self, features):\n#         att = torch.tanh(self.W(features))\n\n#         score = self.V(att)\n\n#         attention_weights = torch.softmax(score, dim=1)\n\n#         context_vector = attention_weights * features\n#         context_vector = torch.sum(context_vector, dim=1)\n\n#         return context_vector\n# MODEL = \"../input/clrp-pytorch-roberta-pretrain-robertalarge/clrp_roberta_large\"\n# class Model(nn.Module):\n#     def __init__(self):\n#         super(Model,self).__init__()\n#         self.roberta = AutoModel.from_pretrained(MODEL)    \n#         #changed attentionHead Dimension from 768 to 1024 by changing model from roberta-base to roberta-large\n#         self.head = AttentionHead(1024,1024,1)\n#         self.dropout = nn.Dropout(0.1)\n#         self.linear = nn.Linear(self.head.out_features,1)\n\n#     def forward(self,**xb):\n#         x = self.roberta(**xb)[0]\n#         x = self.head(x)\n#         return x\n# #ここが新規\n# def get_embeddings(df,path,plot_losses=True, verbose=True):\n#     #cuda使えたら使う構文\n#     device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n#     print(f\"{device} is used\")\n            \n#     model = Model()\n#     model.load_state_dict(torch.load(path))\n#     model.to(device)\n#     model.eval()\n    \n#     tokenizer = AutoTokenizer.from_pretrained(MODEL)\n    \n#     ds = CLRPDataset(df,tokenizer)\n#     dl = DataLoader(ds,\n#                   batch_size = config[\"batch_size\"],\n#                   shuffle=False,\n#                   num_workers = 4,\n#                   pin_memory=True,\n#                   drop_last=False\n#                  )\n        \n#     #以下でpredictionsを抽出するために使った構文を使ってembeddingsをreturnしている.\n#     #SVMの手法とは、embeddingsの意味は？\n#     embeddings = list()\n#     with torch.no_grad():\n#         for i, inputs in tqdm(enumerate(dl)):\n#             inputs = {key:val.reshape(val.shape[0],-1).to(device) for key,val in inputs.items()}\n#             outputs = model(**inputs)\n#             outputs = outputs.detach().cpu().numpy()\n#             embeddings.extend(outputs)\n#     return np.array(embeddings)\n\n# verbose = 0\n# C=10\n# kernel='rbf'\n# #train/testでembeddingsを取得している\n# train_embeddings1 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model0/model0.bin')\n# test_embeddings1 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model0/model0.bin')\n# # model1 = RandomForestRegressor(verbose=verbose,n_jobs=-1)\n# # model1 = LGBMRegressor(n_jobs=-1)\n# model1 = SVR(C=C,kernel=kernel,gamma='auto')\n# model1.fit(train_embeddings1,target)\n# preds1 = model1.predict(test_embeddings1)\n# # print(preds1)\n# del model1,train_embeddings1,test_embeddings1\n# gc.collect()\n\n# train_embeddings2 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model1/model1.bin')\n# test_embeddings2 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model1/model1.bin')\n# model2 = SVR(C=C,kernel=kernel,gamma='auto')\n# model2.fit(train_embeddings2,target)\n# preds2 = model2.predict(test_embeddings2)\n# # print(preds2)\n# del model2,train_embeddings2,test_embeddings2\n# gc.collect()\n\n# train_embeddings3 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model2/model2.bin')\n# test_embeddings3 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model2/model2.bin')\n# model3 = SVR(C=C,kernel=kernel,gamma='auto')\n# model3.fit(train_embeddings3,target)\n# preds3 = model3.predict(test_embeddings3)\n# # print(preds3)\n# del model3,train_embeddings3,test_embeddings3\n# gc.collect()\n\n# train_embeddings4 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model3/model3.bin')\n# test_embeddings4 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model3/model3.bin')\n# model4 = SVR(C=C,kernel=kernel,gamma='auto')\n# model4.fit(train_embeddings4,target)\n# preds4 = model4.predict(test_embeddings4)\n# # print(preds4)\n# del model4,train_embeddings4,test_embeddings4\n# gc.collect()\n\n# train_embeddings5 =  get_embeddings(train_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model4/model4.bin')\n# test_embeddings5 = get_embeddings(test_data,'../input/clrp-pytorch-roberta-finetune-robertalarge/model4/model4.bin')\n# model5 = SVR(C=C,kernel=kernel,gamma='auto')\n# model5.fit(train_embeddings5,target)\n# preds5 = model5.predict(test_embeddings5)\n# # print(preds5)\n# del model5,train_embeddings5,test_embeddings5\n# gc.collect()\n\n# # ml_preds = np.mean([preds1,preds2,preds3,preds4,preds5],axis=0)\n# ml_preds = np.median([preds1,preds2,preds3,preds4,preds5],axis=0)","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:31:46.558172Z","iopub.execute_input":"2021-08-07T10:31:46.558564Z","iopub.status.idle":"2021-08-07T10:31:46.569208Z","shell.execute_reply.started":"2021-08-07T10:31:46.558526Z","shell.execute_reply":"2021-08-07T10:31:46.568444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# [-0.34636177, -0.43033494, -0.54036951, -2.3417344 , -2.01250573, -1.16763746,  0.4171519 ]","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:31:46.570584Z","iopub.execute_input":"2021-08-07T10:31:46.571095Z","iopub.status.idle":"2021-08-07T10:31:46.58001Z","shell.execute_reply.started":"2021-08-07T10:31:46.570934Z","shell.execute_reply":"2021-08-07T10:31:46.579118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Inference","metadata":{}},{"cell_type":"code","source":"# predictions = model1_predictions * 0.5 + model2_predictions * 0.5\n# predictions = np.mean([model1_predictions,np.median(pred_df1,axis=1),np.median(pred_df2,axis=1),np.median(pred_df3,axis=1)],axis=0)\n# predictions = model1_predictions*0.5 + (np.median(pred_df1,axis=1)*0.3 + np.median(pred_df2,axis=1)*0.5 + np.median(pred_df3,axis=1)*0.2 )*0.5\n\n# predictions_final = model1_predictions * 0.45 + model3_predictions * 0.25 + all_preds_4 * 0.2 + ml_preds * 0.1\n# predictions_final = np.mean([model1_predictions,model3_predictions,all_preds_4,ml_preds],axis=0)\n# predictions_final = np.mean([model1_predictions,model3_predictions,all_preds_4,model5_predictions,ml_preds],axis=0)\n# predictions_final = np.mean([model1_predictions,model3_predictions,model5_predictions],axis=0)\n# predictions_final = np.mean([model1_predictions,model3_predictions,model6_predictions],axis=0)\n# predictions_final = model1_predictions * 0.4 + model3_predictions * 0.2 + all_preds_4 * 0.2 + model5_predictions * 0.1 + ml_preds * 0.1\n# predictions_final = 0.5 * model1_predictions + 0.3 * model3_predictions + 0.2 * all_preds_4","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:31:46.58137Z","iopub.execute_input":"2021-08-07T10:31:46.581827Z","iopub.status.idle":"2021-08-07T10:31:46.587895Z","shell.execute_reply.started":"2021-08-07T10:31:46.58179Z","shell.execute_reply":"2021-08-07T10:31:46.587118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions_final = model1_predictions * 0.25 + model3_predictions * 0.25 + all_preds_4 * 0.4 + ml_preds * 0.1\n# predictions_final = model1_predictions * 0.2 + model3_predictions * 0.2 + all_preds_4 * 0.2 + all_preds_5 * 0.3 + ml_preds * 0.1\n# predictions_final = model1_predictions * 0.2 + model3_predictions * 0.2 + all_preds_4 * 0.1 + all_preds_5 * 0.5","metadata":{"execution":{"iopub.status.busy":"2021-08-07T10:31:46.590052Z","iopub.execute_input":"2021-08-07T10:31:46.590478Z","iopub.status.idle":"2021-08-07T10:31:46.598063Z","shell.execute_reply.started":"2021-08-07T10:31:46.590433Z","shell.execute_reply":"2021-08-07T10:31:46.597193Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# submission_df.target = predictions_final\n# submission_df.to_csv(\"submission.csv\", index=False)\n","metadata":{"_uuid":"932f2c25-03ca-46e7-a87f-7daadf0247a6","_cell_guid":"93620de9-b3b8-4129-8b7b-0b276cc58d68","collapsed":false,"jupyter":{"outputs_hidden":false},"execution":{"iopub.status.busy":"2021-08-07T10:31:46.599694Z","iopub.execute_input":"2021-08-07T10:31:46.600166Z","iopub.status.idle":"2021-08-07T10:31:46.606252Z","shell.execute_reply.started":"2021-08-07T10:31:46.600127Z","shell.execute_reply":"2021-08-07T10:31:46.60538Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"db10450f-b0f8-4687-aed6-d1a986657ce6","_cell_guid":"f182401e-add4-489a-97ad-253995c77ea4","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"_uuid":"6116227b-04bd-4c60-988b-e244833da61d","_cell_guid":"ed2be4e8-71fe-4eb2-a4cf-7cc6b30c5519","collapsed":false,"jupyter":{"outputs_hidden":false},"trusted":true},"execution_count":null,"outputs":[]}]}