{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# 导入相关的包\nimport os\nimport gc\nimport cv2\nimport copy\nimport time\nimport random\nimport joblib\nimport numpy as np\nimport pandas as pd\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nfrom tqdm import tqdm","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:45:52.542866Z","iopub.execute_input":"2022-08-10T01:45:52.543325Z","iopub.status.idle":"2022-08-10T01:45:52.550432Z","shell.execute_reply.started":"2022-08-10T01:45:52.543272Z","shell.execute_reply":"2022-08-10T01:45:52.549382Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 一、Cocolm-base","metadata":{}},{"cell_type":"code","source":"# configuration_cocolm.py\n\nimport json\nimport logging\nimport sys\nfrom io import open\n\nfrom transformers.configuration_utils import PretrainedConfig\n\nlogger = logging.getLogger(__name__)\n\nclass COCOLMConfig(PretrainedConfig):\n model_type = \"cocolm\"\n# pretrained_config_archive_map = COCOLM_PRETRAINED_CONFIG_ARCHIVE_MAP\n def __init__(\n self,\n vocab_size=30522,\n embedding_size=128,\n hidden_size=256,\n num_hidden_layers=12,\n num_attention_heads=4,\n intermediate_size=1024,\n hidden_act=\"gelu\",\n hidden_dropout_prob=0.1,\n attention_probs_dropout_prob=0.1,\n max_position_embeddings=512,\n type_vocab_size=2,\n initializer_range=0.02,\n layer_norm_eps=1e-12,\n summary_type=\"first\",\n summary_use_proj=True,\n summary_activation=\"gelu\",\n summary_last_dropout=0.1,\n pad_token_id=0,\n rel_pos_bins=0,\n max_rel_pos=0,\n layer_norm_type='post',\n **kwargs\n ):\n super(COCOLMConfig, self).__init__(**kwargs)\n if isinstance(vocab_size, str) or (sys.version_info[0] == 2\n and isinstance(vocab_size, unicode)):\n with open(vocab_size, \"r\", encoding='utf-8') as reader:\n json_config = json.loads(reader.read())\n for key, value in json_config.items():\n self.__dict__[key] = value\n elif isinstance(vocab_size, int):\n self.vocab_size = vocab_size\n self.embedding_size = embedding_size\n self.hidden_size = hidden_size\n self.num_hidden_layers = num_hidden_layers\n self.num_attention_heads = num_attention_heads\n self.intermediate_size = intermediate_size\n self.hidden_act = hidden_act\n self.hidden_dropout_prob = hidden_dropout_prob\n self.attention_probs_dropout_prob = attention_probs_dropout_prob\n self.max_position_embeddings = max_position_embeddings\n self.type_vocab_size = type_vocab_size\n self.initializer_range = initializer_range\n self.layer_norm_eps = layer_norm_eps\n\n self.summary_type = summary_type\n self.summary_use_proj = summary_use_proj\n self.summary_activation = summary_activation\n self.summary_last_dropout = summary_last_dropout\n self.rel_pos_bins = rel_pos_bins\n self.max_rel_pos = max_rel_pos\n self.layer_norm_type = layer_norm_type\n else:\n raise ValueError(\"First argument must be either a vocabulary size (int)\"\n \" or the path to a pretrained model config file (str)\")\n \n @classmethod\n def from_pretrained(cls, pretrained_model_name_or_path, **kwargs):\n print(f'from_pretrained:{pretrained_model_name_or_path}')\n return super().from_pretrained(pretrained_model_name_or_path, **kwargs)\n\n \n# config_cocolm = COCOLMConfig.from_pretrained(\"microsoft/cocolm-base\")\nconfig_cocolm = COCOLMConfig.from_pretrained(\"../input/cocobase/cocolm-base/config.json\")\nprint(config_cocolm)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:45:53.660391Z","iopub.execute_input":"2022-08-10T01:45:53.660793Z","iopub.status.idle":"2022-08-10T01:45:53.679617Z","shell.execute_reply.started":"2022-08-10T01:45:53.660764Z","shell.execute_reply":"2022-08-10T01:45:53.678499Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# convert_state_dict.py\n\nimport torch\nimport logging\n\nfrom transformers.modeling_utils import cached_path, WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME\n\nlogger = logging.getLogger(__name__)\n\n\ndef get_checkpoint_from_transformer_cache(\n archive_file, pretrained_model_name_or_path, pretrained_model_archive_map,\n cache_dir, force_download, proxies, resume_download,\n):\n try:\n resolved_archive_file = cached_path(archive_file, cache_dir=cache_dir, force_download=force_download,\n proxies=proxies, resume_download=resume_download)\n except EnvironmentError:\n if pretrained_model_name_or_path in pretrained_model_archive_map:\n msg = \"Couldn't reach server at '{}' to download pretrained weights.\".format(\n archive_file)\n else:\n msg = \"Model name '{}' was not found in model name list ({}). \" \\\n \"We assumed '{}' was a path or url to model weight files named one of {} but \" \\\n \"couldn't find any such file at this path or url.\".format(\n pretrained_model_name_or_path,\n ', '.join(pretrained_model_archive_map.keys()),\n archive_file,\n [WEIGHTS_NAME, TF2_WEIGHTS_NAME, TF_WEIGHTS_NAME])\n raise EnvironmentError(msg)\n\n if resolved_archive_file == archive_file:\n logger.info(\"loading weights file {}\".format(archive_file))\n else:\n logger.info(\"loading weights file {} from cache at {}\".format(\n archive_file, resolved_archive_file))\n\n return torch.load(resolved_archive_file, map_location='cpu')\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:45:54.482186Z","iopub.execute_input":"2022-08-10T01:45:54.482937Z","iopub.status.idle":"2022-08-10T01:45:54.491729Z","shell.execute_reply.started":"2022-08-10T01:45:54.482899Z","shell.execute_reply":"2022-08-10T01:45:54.490070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# modeling_cocolm.py\n\nimport logging\nimport math\nimport os\n\nimport torch\nfrom torch import nn\nfrom torch.nn.modules.loss import _Loss\nimport torch.nn.functional as F\nfrom torch.nn import Parameter\n\nfrom transformers.modeling_utils import PreTrainedModel, PoolerAnswerClass, PoolerEndLogits, PoolerStartLogits\nfrom transformers.models.bert.modeling_bert import ACT2FN\nfrom transformers.file_utils import WEIGHTS_NAME\nfrom transformers.activations import get_activation\n\n# from cocolm.configuration_cocolm import COCOLMConfig\n# from cocolm.convert_state_dict import get_checkpoint_from_transformer_cache\n\nlogger = logging.getLogger(__name__)\n\ntry:\n from apex.normalization.fused_layer_norm import FusedLayerNorm as COCOLMLayerNorm\nexcept ImportError:\n print(\"Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.\")\n from torch.nn import LayerNorm as COCOLMLayerNorm\n\n\nCOCOLM_PRETRAINED_MODEL_ARCHIVE_MAP = {\n 'microsoft/cocolm-base': \"https://huggingface.co/microsoft/cocolm-base/resolve/main/pytorch_model.bin\",\n 'microsoft/cocolm-large': \"https://huggingface.co/microsoft/cocolm-large/resolve/main/pytorch_model.bin\",\n}\n\nclass COCOLMPreTrainedModel(PreTrainedModel):\n \"\"\" An abstract class to handle weights initialization \n and a simple interface for dowloading and loading pretrained models.\n \"\"\"\n config_class = COCOLMConfig\n supported_convert_pretrained_model_archive_map = {\n \"cocolm\": COCOLM_PRETRAINED_MODEL_ARCHIVE_MAP,\n }\n base_model_prefix = \"cocolm\"\n pretrained_model_archive_map = {\n **COCOLM_PRETRAINED_MODEL_ARCHIVE_MAP,\n }\n\n def _init_weights(self, module):\n \"\"\" Initialize the weights \"\"\"\n if isinstance(module, (nn.Linear, nn.Embedding)):\n # Slightly different from the TF version which uses truncated_normal for initialization\n # cf https://github.com/pytorch/pytorch/pull/5617\n module.weight.data.normal_(\n mean=0.0, std=self.config.initializer_range)\n elif isinstance(module, COCOLMLayerNorm):\n module.bias.data.zero_()\n module.weight.data.fill_(1.0)\n if isinstance(module, nn.Linear) and module.bias is not None:\n module.bias.data.zero_()\n\n @classmethod\n def from_pretrained(\n cls, pretrained_model_name_or_path, reuse_position_embedding=True,\n drop_parameters=None, *model_args, **kwargs,\n ):\n model_type = kwargs.pop('model_type', 'cocolm')\n if model_type is not None and \"state_dict\" not in kwargs:\n if model_type in cls.supported_convert_pretrained_model_archive_map:\n pretrained_model_archive_map = cls.supported_convert_pretrained_model_archive_map[\n model_type]\n if pretrained_model_name_or_path in pretrained_model_archive_map:\n state_dict = get_checkpoint_from_transformer_cache(\n archive_file=pretrained_model_archive_map[pretrained_model_name_or_path],\n pretrained_model_name_or_path=pretrained_model_name_or_path,\n pretrained_model_archive_map=pretrained_model_archive_map,\n cache_dir=kwargs.get(\"cache_dir\", None), force_download=kwargs.get(\"force_download\", None),\n proxies=kwargs.get(\"proxies\", None), resume_download=kwargs.get(\"resume_download\", None),\n )\n kwargs[\"state_dict\"] = state_dict\n logger.info(\"Load HF ckpts\")\n elif os.path.isfile(pretrained_model_name_or_path):\n state_dict = torch.load(\n pretrained_model_name_or_path, map_location='cpu')\n kwargs[\"state_dict\"] = state_dict\n logger.info(\"Load local ckpts\")\n elif os.path.isdir(pretrained_model_name_or_path):\n state_dict = torch.load(os.path.join(\n pretrained_model_name_or_path, WEIGHTS_NAME), map_location='cpu')\n kwargs[\"state_dict\"] = state_dict\n logger.info(\"Load local ckpts\")\n else:\n raise RuntimeError(\n \"No pre-trained checkpoint !\")\n\n if kwargs[\"state_dict\"] is None:\n logger.info(\"s2s-ft does't support the model !\")\n raise NotImplementedError()\n\n state_dict = kwargs[\"state_dict\"]\n _k = 'cocolm.embeddings.position_embeddings.weight'\n if _k in state_dict and \"config\" in kwargs:\n config = kwargs[\"config\"]\n if config.max_position_embeddings > state_dict[_k].shape[0]:\n logger.info(\"Resize > position embeddings !\")\n old_vocab_size = state_dict[_k].shape[0]\n new_postion_embedding = state_dict[_k].data.new_tensor(torch.ones(\n size=(config.max_position_embeddings, state_dict[_k].shape[1])), dtype=torch.float)\n new_postion_embedding = nn.Parameter(\n data=new_postion_embedding, requires_grad=True)\n new_postion_embedding.data.normal_(\n mean=0.0, std=config.initializer_range)\n max_range = config.max_position_embeddings if reuse_position_embedding else old_vocab_size\n shift = 0\n while shift < max_range:\n delta = min(old_vocab_size, max_range - shift)\n new_postion_embedding.data[shift: shift +\n delta, :] = state_dict[_k][:delta, :]\n logger.info(\" CP [%d ~ %d] into [%d ~ %d] \" %\n (0, delta, shift, shift + delta))\n shift += delta\n state_dict[_k] = new_postion_embedding.data\n del new_postion_embedding\n elif config.max_position_embeddings < state_dict[_k].shape[0]:\n logger.info(\"Resize < position embeddings !\")\n old_vocab_size = state_dict[_k].shape[0]\n new_postion_embedding = state_dict[_k].data.new_tensor(torch.ones(\n size=(config.max_position_embeddings, state_dict[_k].shape[1])), dtype=torch.float)\n new_postion_embedding = nn.Parameter(\n data=new_postion_embedding, requires_grad=True)\n new_postion_embedding.data.normal_(\n mean=0.0, std=config.initializer_range)\n new_postion_embedding.data.copy_(\n state_dict[_k][:config.max_position_embeddings, :])\n state_dict[_k] = new_postion_embedding.data\n del new_postion_embedding\n\n if drop_parameters is not None:\n if not isinstance(drop_parameters, list):\n raise RuntimeError()\n not_drop_state_dict = {}\n for key in state_dict:\n drop_flag = False\n for prefix in drop_parameters:\n if key.startswith(prefix):\n drop_flag = True\n break\n\n if drop_flag:\n logger.info(\"Drop %s\" % key)\n else:\n not_drop_state_dict[key] = state_dict[key]\n\n kwargs[\"state_dict\"] = not_drop_state_dict\n del state_dict\n if pretrained_model_name_or_path in pretrained_model_archive_map:\n pretrained_model_name_or_path = pretrained_model_archive_map[pretrained_model_name_or_path]\n elif not os.path.isfile(pretrained_model_name_or_path):\n pretrained_model_name_or_path = 'microsoft/cocolm-large' \n \n print(f'from_pretrained:{pretrained_model_name_or_path}')\n return super().from_pretrained(pretrained_model_name_or_path, *model_args, **kwargs)\n\n\nclass COCOLMEmbeddings(nn.Module):\n \"\"\"Construct the embeddings from word, position and token_type embeddings.\n \"\"\"\n\n def __init__(self, config):\n super(COCOLMEmbeddings, self).__init__()\n self.word_embeddings = nn.Embedding(\n config.vocab_size, config.hidden_size, padding_idx=0)\n fix_word_embedding = getattr(config, \"fix_word_embedding\", None)\n if fix_word_embedding:\n self.word_embeddings.weight.requires_grad = False\n self.position_embeddings = nn.Embedding(\n config.max_position_embeddings, config.hidden_size)\n if config.type_vocab_size > 0:\n self.token_type_embeddings = nn.Embedding(\n config.type_vocab_size, config.hidden_size)\n else:\n self.token_type_embeddings = None\n\n # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load\n # any TensorFlow checkpoint file\n self.layer_norm_type = config.layer_norm_type\n if self.layer_norm_type in ('post',):\n self.LayerNorm = COCOLMLayerNorm(\n config.hidden_size, eps=config.layer_norm_eps)\n self.dropout = nn.Dropout(config.hidden_dropout_prob)\n\n def forward(self, input_ids=None, token_type_ids=None, position_ids=None, inputs_embeds=None):\n if input_ids is not None:\n input_shape = input_ids.size()\n else:\n input_shape = inputs_embeds.size()[:-1]\n\n seq_length = input_shape[1]\n device = input_ids.device if input_ids is not None else inputs_embeds.device\n if position_ids is None:\n position_ids = torch.arange(\n seq_length, dtype=torch.long, device=device)\n position_ids = position_ids.unsqueeze(0).expand(input_shape)\n if token_type_ids is None:\n token_type_ids = torch.zeros(\n input_shape, dtype=torch.long, device=device)\n\n if inputs_embeds is None:\n inputs_embeds = self.word_embeddings(input_ids)\n position_embeddings = self.position_embeddings(position_ids)\n\n embeddings = inputs_embeds + position_embeddings\n\n if self.token_type_embeddings:\n embeddings = embeddings + \\\n self.token_type_embeddings(token_type_ids)\n\n if self.layer_norm_type in ('post',):\n embeddings = self.LayerNorm(embeddings)\n embeddings = self.dropout(embeddings)\n return embeddings, position_ids\n\n\nclass SelfMultiheadAttention(nn.Module):\n \"\"\"Multi-headed attention.\n\n See \"Attention Is All You Need\" for more details.\n \"\"\"\n\n def __init__(\n self,\n embed_dim,\n num_heads,\n dropout=0.0,\n bias=True,\n scaling_factor=1,\n ):\n super().__init__()\n self.embed_dim = embed_dim\n\n self.num_heads = num_heads\n self.dropout = nn.Dropout(dropout)\n\n self.head_dim = embed_dim // num_heads\n assert (\n self.head_dim * num_heads == self.embed_dim\n ), \"embed_dim must be divisible by num_heads\"\n self.scaling = (self.head_dim * scaling_factor) ** -0.5\n\n self.qk_head_dim = self.head_dim\n self.in_proj = nn.Linear(embed_dim, embed_dim * 3, bias=bias)\n\n self.out_proj = nn.Linear(embed_dim, embed_dim, bias=bias)\n\n def forward(\n self,\n query,\n key_padding_mask,\n need_weights,\n attn_mask,\n attn_bias,\n before_softmax=False,\n need_head_weights=False,\n ):\n \"\"\"Input shape: Time x Batch x Channel\n\n Args:\n key_padding_mask (ByteTensor, optional): mask to exclude\n keys that are pads, of shape `(batch, src_len)`, where\n padding elements are indicated by 1s.\n need_weights (bool, optional): return the attention weights,\n averaged over heads (default: False).\n attn_mask (ByteTensor, optional): typically used to\n implement causal attention, where the mask prevents the\n attention from looking forward in time (default: None).\n before_softmax (bool, optional): return the raw attention\n weights and values before the attention softmax.\n need_head_weights (bool, optional): return the attention\n weights for each head. Implies *need_weights*. Default:\n return the average attention weights over all heads.\n \"\"\"\n if need_head_weights:\n need_weights = True\n\n tgt_len, bsz, embed_dim = query.size()\n assert embed_dim == self.embed_dim\n assert list(query.size()) == [tgt_len, bsz, embed_dim]\n\n if self.qk_head_dim == self.head_dim:\n # , self.k_proj(query), self.v_proj(query)\n q, k, v = self.in_proj(query).chunk(3, dim=-1)\n else:\n q, k = self.qk_proj(query).chunk(2, dim=-1)\n v = self.v_proj(query)\n\n q = (\n q.contiguous().view(tgt_len, bsz * self.num_heads, self.qk_head_dim)\n .transpose(0, 1) * self.scaling\n )\n if k is not None:\n k = (\n k.contiguous().view(-1, bsz * self.num_heads, self.qk_head_dim)\n .transpose(0, 1)\n )\n if v is not None:\n v = (\n v.contiguous().view(-1, bsz * self.num_heads, self.head_dim)\n .transpose(0, 1)\n )\n\n assert k is not None\n src_len = k.size(1)\n\n # This is part of a workaround to get around fork/join parallelism\n # not supporting Optional types.\n if key_padding_mask is not None and key_padding_mask.dim() == 0:\n key_padding_mask = None\n\n if key_padding_mask is not None:\n assert key_padding_mask.size(0) == bsz\n assert key_padding_mask.size(1) == src_len\n\n attn_weights = torch.bmm(q, k.transpose(1, 2))\n\n assert list(attn_weights.size()) == [bsz * self.num_heads, tgt_len, src_len]\n\n if attn_mask is not None:\n attn_mask = attn_mask.unsqueeze(0)\n attn_weights += attn_mask\n\n if key_padding_mask is not None:\n # don't attend to padding symbols\n attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len)\n attn_weights.masked_fill_(\n key_padding_mask.unsqueeze(1).unsqueeze(2).to(torch.bool),\n float(\"-inf\")\n )\n attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len)\n\n if attn_bias is not None:\n attn_weights += attn_bias\n\n if before_softmax:\n return attn_weights, v\n\n attn_probs = nn.Softmax(dim=-1)(attn_weights)\n attn_probs = self.dropout(attn_probs)\n\n assert v is not None\n attn = torch.bmm(attn_probs, v)\n assert list(attn.size()) == [bsz * self.num_heads, tgt_len, self.head_dim]\n attn = attn.transpose(0, 1).contiguous().view(tgt_len, bsz, embed_dim)\n attn = self.out_proj(attn)\n ret_attn_weights = None\n if need_weights:\n ret_attn_weights = attn_weights.view(\n bsz, self.num_heads, tgt_len, src_len\n ).transpose(1, 0)\n if not need_head_weights:\n # average attention weights over heads\n ret_attn_weights = attn_weights.mean(dim=0)\n\n return attn, ret_attn_weights\n\n\nclass COCOLMAttention(nn.Module):\n def __init__(self, config):\n super().__init__()\n self.config = config\n self.self_attn = SelfMultiheadAttention(config.hidden_size, config.num_attention_heads,\n dropout=config.attention_probs_dropout_prob)\n self.dropout = nn.Dropout(config.attention_probs_dropout_prob)\n self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)\n\n def forward(self, x, attention_mask, attn_bias):\n x = x.transpose(0, 1)\n residual = x\n x, attn = self.self_attn(query=x, key_padding_mask=attention_mask, need_weights=True if self.config.output_attentions else False, attn_mask=None, attn_bias=attn_bias)\n x[x != x] = 0\n x = self.dropout(x)\n x = residual + x\n x = self.LayerNorm(x)\n if attn is not None:\n return (x.transpose(0, 1), ) + attn\n return (x.transpose(0, 1), )\n\n\nclass COCOLMIntermediate(nn.Module):\n def __init__(self, config):\n super().__init__()\n self.dense = nn.Linear(config.hidden_size, config.intermediate_size)\n if isinstance(config.hidden_act, str):\n self.intermediate_act_fn = ACT2FN[config.hidden_act]\n else:\n self.intermediate_act_fn = config.hidden_act\n\n def forward(self, hidden_states):\n hidden_states = self.dense(hidden_states)\n hidden_states = self.intermediate_act_fn(hidden_states)\n return hidden_states\n\n\nclass COCOLMOutput(nn.Module):\n def __init__(self, config):\n super().__init__()\n self.dense = nn.Linear(config.intermediate_size, config.hidden_size)\n self.layer_norm_type = config.layer_norm_type\n if self.layer_norm_type in ('post', 'hybrid'):\n self.LayerNorm = COCOLMLayerNorm(\n config.hidden_size, eps=config.layer_norm_eps)\n self.dropout = nn.Dropout(config.hidden_dropout_prob)\n\n def forward(self, hidden_states, input_tensor):\n hidden_states = self.dense(hidden_states)\n hidden_states = self.dropout(hidden_states)\n if self.layer_norm_type == 'pre':\n return hidden_states + input_tensor\n elif self.layer_norm_type == 'hybrid':\n return hidden_states + self.LayerNorm(input_tensor) + input_tensor\n else:\n return self.LayerNorm(hidden_states + input_tensor)\n\n\nclass COCOLMLayer(nn.Module):\n def __init__(self, config):\n super(COCOLMLayer, self).__init__()\n self.attention = COCOLMAttention(config)\n if hasattr(config, 'num_ffn_layers') and config.num_ffn_layers > 1:\n self.num_ffn_layers = config.num_ffn_layers\n self.intermediate = nn.ModuleList(\n [COCOLMIntermediate(config) for _ in range(config.num_ffn_layers)])\n self.output = nn.ModuleList(\n [COCOLMOutput(config) for _ in range(config.num_ffn_layers)])\n else:\n self.num_ffn_layers = 0\n self.intermediate = COCOLMIntermediate(config)\n self.output = COCOLMOutput(config)\n\n self.layer_norm_type = config.layer_norm_type\n if self.layer_norm_type in ('pre', 'hybrid'):\n if self.num_ffn_layers > 0:\n self.LayerNorm = nn.ModuleList([COCOLMLayerNorm(\n config.hidden_size, eps=1e-5) for i in range(self.num_ffn_layers)])\n else:\n self.LayerNorm = COCOLMLayerNorm(config.hidden_size, eps=1e-5)\n\n def forward(self, hidden_states, attention_mask=None, split_lengths=None, rel_pos=None):\n self_attention_outputs = self.attention(hidden_states, attention_mask, rel_pos)\n attention_output = self_attention_outputs[0]\n\n if isinstance(self.intermediate, nn.ModuleList):\n layer_output = attention_output\n for i, (intermediate_layer, output_layer) in enumerate(zip(self.intermediate, self.output)):\n if self.layer_norm_type in ('pre', 'hybrid'):\n _attention_output = self.LayerNorm[i](layer_output)\n else:\n _attention_output = layer_output\n intermediate_output = intermediate_layer(_attention_output)\n layer_output = output_layer(intermediate_output, layer_output)\n else:\n if self.layer_norm_type in ('pre', 'hybrid'):\n _attention_output = self.LayerNorm(attention_output)\n else:\n _attention_output = attention_output\n intermediate_output = self.intermediate(_attention_output)\n layer_output = self.output(intermediate_output, attention_output)\n\n outputs = (layer_output,) + self_attention_outputs[1:]\n return outputs\n\n\nclass COCOLMEncoder(nn.Module):\n def __init__(self, config):\n super(COCOLMEncoder, self).__init__()\n self.output_attentions = config.output_attentions\n self.output_hidden_states = config.output_hidden_states\n self.layer = nn.ModuleList([COCOLMLayer(config)\n for _ in range(config.num_hidden_layers)])\n self.layer_norm_type = config.layer_norm_type\n if self.layer_norm_type in ('pre', 'hybrid'):\n self.LayerNorm = COCOLMLayerNorm(config.hidden_size, eps=1e-5)\n\n def forward(self, hidden_states, attention_mask=None, split_lengths=None, rel_pos=None):\n all_hidden_states = ()\n all_attentions = ()\n for i, layer_module in enumerate(self.layer):\n if self.output_hidden_states:\n all_hidden_states = all_hidden_states + (hidden_states,)\n\n layer_outputs = layer_module(\n hidden_states, attention_mask,\n split_lengths=split_lengths, rel_pos=rel_pos)\n hidden_states = layer_outputs[0]\n\n if (self.layer_norm_type in ('pre', 'hybrid')) and (i == len(self.layer) - 1):\n # pre-layernorm: apply layernorm for the topmost hidden states\n hidden_states = self.LayerNorm(hidden_states)\n\n if self.output_attentions:\n all_attentions = all_attentions + (layer_outputs[1],)\n\n # Add last layer\n if self.output_hidden_states:\n all_hidden_states = all_hidden_states + (hidden_states,)\n\n outputs = (hidden_states,)\n if self.output_hidden_states:\n outputs = outputs + (all_hidden_states,)\n if self.output_attentions:\n outputs = outputs + (all_attentions,)\n # last-layer hidden state, (all hidden states), (all attentions)\n return outputs\n\n\nclass COCOLMBinaryPredictions(nn.Module):\n \"\"\"Binary prediction module for the main model.\"\"\"\n\n def __init__(self, config):\n super().__init__()\n self.out_proj = nn.Linear(config.hidden_size, 1)\n self.config = config\n\n def forward(self, hidden_states):\n logits = self.out_proj(hidden_states).squeeze(-1)\n return logits\n\n\nclass COCOLMCLMHead(nn.Module):\n def __init__(self, config):\n super(COCOLMCLMHead, self).__init__()\n # The output weights are the same as the input embeddings, but there is\n # an output-only bias for each token.\n self.decoder = nn.Linear(config.hidden_size, config.vocab_size, bias=False)\n self.bias = nn.Parameter(torch.zeros(config.vocab_size))\n self.decoder.bias = self.bias\n\n def forward(self, hidden_states):\n x = self.decoder(hidden_states)\n return x\n\n\nclass COCOLMSCLHead(nn.Module):\n def __init__(self, config):\n super().__init__()\n self.dense = nn.Linear(config.hidden_size, config.hidden_size)\n self.activation = get_activation(\"gelu\")\n self.LayerNorm = COCOLMLayerNorm(\n config.hidden_size, eps=config.layer_norm_eps)\n\n def forward(self, hidden_states):\n # We \"pool\" the model by simply taking the hidden state corresponding\n # to the first token.\n first_token_tensor = hidden_states[:, 0]\n pooled_output = self.dense(first_token_tensor)\n pooled_output = self.activation(pooled_output)\n pooled_output = self.LayerNorm(pooled_output)\n return pooled_output\n\n\ndef relative_position_bucket(relative_position, num_buckets=32, max_distance=128):\n sign = torch.sign(relative_position)\n num_buckets //= 2\n n = torch.abs(relative_position)\n\n # half of the buckets are for exact increments in positions\n max_exact = num_buckets // 2\n is_small = n < max_exact\n max_bucket_val = num_buckets - 1 - max_exact\n # The other half of the buckets are for logarithmically bigger bins in positions up to max_distance\n val_if_large = max_exact + torch.ceil(\n torch.log(n.float() / max_exact) / math.log((max_distance - 1) / max_exact) * (max_bucket_val)\n ).long()\n val_if_large = torch.min(val_if_large, torch.full_like(val_if_large, num_buckets - 1))\n ret = torch.where(is_small, n, val_if_large) * sign\n return ret\n\n\nclass COCOLMModel(COCOLMPreTrainedModel):\n\n def __init__(self, config):\n super(COCOLMModel, self).__init__(config)\n self.config = config\n\n self.embeddings = COCOLMEmbeddings(config)\n self.encoder = COCOLMEncoder(config)\n \n if hasattr(config, 'need_pooler') and getattr(config, 'need_pooler'):\n self.scl_head = COCOLMSCLHead(config)\n else:\n self.scl_head = None\n\n if self.config.rel_pos_bins > 0:\n assert self.config.rel_pos_bins % 2 == 0\n self.relative_attention_bias = nn.Embedding(self.config.rel_pos_bins, self.config.num_attention_heads)\n context_position = torch.arange(self.config.max_position_embeddings, dtype=torch.long)[:, None]\n memory_position = torch.arange(self.config.max_position_embeddings, dtype=torch.long)[None, :]\n relative_position = memory_position - context_position\n self.rp_bucket = relative_position_bucket(\n relative_position,\n num_buckets=self.config.rel_pos_bins,\n max_distance=self.config.max_rel_pos\n )\n self.rp_bucket -= self.rp_bucket.min()\n\n def get_rel_pos_bias(self, x):\n # Assume the input is ordered. If your input token is permuted, you may need to update this accordingly\n if self.rp_bucket.device != x.device:\n self.rp_bucket = self.rp_bucket.to(x.device)\n seq_len = x.size(1)\n rp_bucket = self.rp_bucket[:seq_len, :seq_len]\n values = F.embedding(rp_bucket, self.relative_attention_bias.weight)\n values = values.permute([2, 0, 1])\n return values.contiguous()\n\n def forward(self, input_ids=None, attention_mask=None, token_type_ids=None,\n position_ids=None, inputs_embeds=None, split_lengths=None):\n if input_ids is not None and inputs_embeds is not None:\n raise ValueError(\n \"You cannot specify both input_ids and inputs_embeds at the same time\")\n elif input_ids is not None:\n input_shape = input_ids.size()\n elif inputs_embeds is not None:\n input_shape = inputs_embeds.size()[:-1]\n else:\n raise ValueError(\n \"You have to specify either input_ids or inputs_embeds\")\n\n device = input_ids.device if input_ids is not None else inputs_embeds.device\n\n if attention_mask is None:\n attention_mask = torch.ones(input_shape, device=device)\n\n # Since attention_mask is 1.0 for positions we want to attend and 0.0 for\n # masked positions, this operation will create a tensor which is 0.0 for\n # positions we want to attend and -10000.0 for masked positions.\n # Since we are adding it to the raw scores before the softmax, this is\n # effectively the same as removing these entirely.\n extended_attention_mask = attention_mask == 0\n\n embedding_output, position_ids = self.embeddings(\n input_ids=input_ids, position_ids=position_ids, token_type_ids=token_type_ids, inputs_embeds=inputs_embeds)\n\n embedding_output = embedding_output * (attention_mask.unsqueeze(-1).type_as(embedding_output))\n\n rel_pos_bias = self.get_rel_pos_bias(input_ids).repeat(input_ids.size(0), 1, 1) if self.config.rel_pos_bins > 0 else None\n seq_len = input_ids.size(1)\n\n if rel_pos_bias is not None and extended_attention_mask is not None:\n # merge key_padding_mask and attn_mask\n rel_pos_bias = rel_pos_bias.view(input_ids.size(0), -1, seq_len, seq_len)\n rel_pos_bias.masked_fill_(\n extended_attention_mask.unsqueeze(1).unsqueeze(2),\n float(\"-inf\")\n )\n rel_pos_bias = rel_pos_bias.view(-1, seq_len, seq_len)\n extended_attention_mask = None\n\n encoder_outputs = self.encoder(\n embedding_output, attention_mask=extended_attention_mask,\n split_lengths=split_lengths, rel_pos=rel_pos_bias)\n sequence_output = encoder_outputs[0]\n\n # add hidden_states and attentions if they are here\n outputs = (sequence_output, ) + encoder_outputs[1:]\n if self.scl_head is None:\n # sequence_output, pooled_output, (hidden_states), (attentions)\n return outputs\n else:\n pooled_output = self.scl_head(sequence_output)\n return sequence_output, pooled_output\n\n\nclass COCOLMClassificationHead(nn.Module):\n \"\"\"Head for sentence-level classification tasks.\"\"\"\n\n def __init__(self, config):\n super().__init__()\n self.dense = nn.Linear(config.hidden_size, config.hidden_size)\n self.dropout = nn.Dropout(config.cls_dropout_prob)\n self.out_proj = nn.Linear(config.hidden_size, config.num_labels)\n\n def forward(self, features, **kwargs):\n x = features[:, 0, :] # take token (equiv. to [CLS])\n x = self.dropout(x)\n x = self.dense(x)\n x = get_activation(\"gelu\")(x)\n x = self.dropout(x)\n x = self.out_proj(x)\n return x\n\n\nclass COCOLMForSequenceClassification(COCOLMPreTrainedModel):\n def __init__(self, config):\n super().__init__(config)\n self.config = config\n self.num_labels = config.num_labels\n\n self.cocolm = COCOLMModel(config)\n self.classifier = COCOLMClassificationHead(config)\n\n self.init_weights()\n\n def forward(\n self,\n input_ids=None,\n attention_mask=None,\n token_type_ids=None,\n position_ids=None,\n inputs_embeds=None,\n labels=None,\n ):\n outputs = self.cocolm(\n input_ids,\n attention_mask=attention_mask,\n token_type_ids=token_type_ids,\n position_ids=position_ids,\n inputs_embeds=inputs_embeds,\n )\n\n logits = self.classifier(outputs[0])\n\n # add hidden states and attention if they are here\n outputs = (logits,) + outputs[2:]\n\n if labels is not None:\n if self.num_labels == 1:\n # We are doing regression\n loss_fct = nn.MSELoss()\n loss = loss_fct(logits.view(-1), labels.view(-1))\n else:\n loss_fct = nn.CrossEntropyLoss()\n loss = loss_fct(\n logits.view(-1, self.num_labels), labels.view(-1))\n outputs = (loss,) + outputs\n\n return outputs # (loss), logits, (hidden_states), (attentions)\n\n\nclass PoolerLogits(nn.Module):\n \"\"\"\n Compute SQuAD start logits from sequence hidden states.\n\n Args:\n config (:class:`~transformers.PretrainedConfig`):\n The config used by the model, will be used to grab the :obj:`hidden_size` of the model.\n \"\"\"\n\n def __init__(self, hidden_size):\n super().__init__()\n self.dense = nn.Linear(hidden_size, 1)\n self.dense.weight.data.normal_(mean=0.0, std=0.02)\n self.dense.bias.data.zero_()\n\n def forward(\n self, hidden_states, p_mask = None\n ):\n \"\"\"\n Args:\n hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_len, hidden_size)`):\n The final hidden states of the model.\n p_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_len)`, `optional`):\n Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token\n should be masked.\n\n Returns:\n :obj:`torch.FloatTensor`: The start logits for SQuAD.\n \"\"\"\n x = self.dense(hidden_states).squeeze(-1)\n\n if p_mask is not None:\n x.masked_fill_(p_mask, float('-inf'))\n return x\n\n\nclass SQuADHead(nn.Module):\n\n def __init__(self, hidden_size):\n super().__init__()\n self.start_logits = PoolerLogits(hidden_size)\n self.end_logits = PoolerLogits(hidden_size)\n\n def forward(\n self,\n hidden_states,\n start_positions=None,\n end_positions=None,\n p_mask = None,\n ):\n \"\"\"\n Args:\n hidden_states (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_len, hidden_size)`):\n Final hidden states of the model on the sequence tokens.\n start_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`):\n Positions of the first token for the labeled span.\n end_positions (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`):\n Positions of the last token for the labeled span.\n is_impossible (:obj:`torch.LongTensor` of shape :obj:`(batch_size,)`, `optional`):\n Whether the question has a possible answer in the paragraph or not.\n p_mask (:obj:`torch.FloatTensor` of shape :obj:`(batch_size, seq_len)`, `optional`):\n Mask for tokens at invalid position, such as query and special symbols (PAD, SEP, CLS). 1.0 means token\n should be masked.\n\n Returns:\n \"\"\"\n start_logits = self.start_logits(hidden_states, p_mask=p_mask)\n end_logits = self.end_logits(hidden_states, p_mask=p_mask)\n\n if start_positions is not None and end_positions is not None:\n def loss_fct(logits, targets):\n return F.nll_loss(\n F.log_softmax(\n logits.view(-1, logits.size(-1)),\n dim=-1,\n dtype=torch.float32,\n ),\n targets.view(-1),\n reduction='sum',\n )\n start_loss = loss_fct(start_logits, start_positions)\n end_loss = loss_fct(end_logits, end_positions)\n total_loss = (start_loss + end_loss) * 0.5\n return total_loss\n else:\n return start_logits, end_logits\n\n\nclass COCOLMForQuestionAnswering(COCOLMPreTrainedModel):\n\n def __init__(self, config):\n super(COCOLMForQuestionAnswering, self).__init__(config)\n self.num_labels = config.num_labels\n \n self.cocolm = COCOLMModel(config)\n self.qa_outputs = SQuADHead(config.hidden_size)\n\n self.init_weights()\n\n def forward(self, input_ids=None, attention_mask=None, token_type_ids=None, position_ids=None, inputs_embeds=None,\n start_positions=None, end_positions=None):\n\n outputs = self.cocolm(input_ids,\n attention_mask=attention_mask,\n token_type_ids=token_type_ids,\n position_ids=position_ids,\n inputs_embeds=inputs_embeds)\n\n sequence_output = outputs[0]\n\n squad_outputs = self.qa_outputs(sequence_output, start_positions, end_positions)\n outputs = (squad_outputs,) + outputs[2:]\n\n return outputs","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:45:55.566668Z","iopub.execute_input":"2022-08-10T01:45:55.567027Z","iopub.status.idle":"2022-08-10T01:45:55.684751Z","shell.execute_reply.started":"2022-08-10T01:45:55.566996Z","shell.execute_reply":"2022-08-10T01:45:55.683547Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model_cocolm = COCOLMModel.from_pretrained(\"microsoft/cocolm-base\", config=config_cocolm)\nmodel_cocolm = COCOLMModel.from_pretrained(\"../input/cocobase/cocolm-base/pytorch_model.bin\", config=config_cocolm)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:45:55.736351Z","iopub.execute_input":"2022-08-10T01:45:55.737098Z","iopub.status.idle":"2022-08-10T01:45:58.494633Z","shell.execute_reply.started":"2022-08-10T01:45:55.737059Z","shell.execute_reply":"2022-08-10T01:45:58.493450Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenization_utils.py\n\n\nimport os\nimport logging\nfrom collections import Counter\n\nimport torch\n\nlogger = logging.getLogger(__name__) # pylint: disable=invalid-name\n\nclass Dictionary:\n \"\"\"A mapping from symbols to consecutive integers\"\"\"\n\n def __init__(\n self,\n *, # begin keyword-only arguments\n bos=\"\",\n pad=\"\",\n eos=\"\",\n unk=\"\",\n extra_special_symbols=None,\n ):\n self.bos_word, self.unk_word, self.pad_word, self.eos_word = bos, unk, pad, eos\n self.symbols = []\n self.count = []\n self.indices = {}\n self.alias_mapper = {'': '[CLS]', '': '[PAD]', '':'[SEP]', '': '[UNK]', '': '[MASK]', \n '[CLS]': '[CLS]', '[PAD]': '[PAD]', '[SEP]':'[SEP]', '[UNK]': '[UNK]', '[MASK]': '[MASK]'}\n self.bos_index = self.add_symbol(bos)\n self.pad_index = self.add_symbol(pad)\n self.eos_index = self.add_symbol(eos)\n self.unk_index = self.add_symbol(unk)\n if extra_special_symbols:\n for s in extra_special_symbols:\n self.add_symbol(s)\n self.nspecial = len(self.symbols)\n\n def __eq__(self, other):\n return self.indices == other.indices\n\n def __getitem__(self, idx):\n if idx < len(self.symbols):\n return self.symbols[idx]\n return self.unk_word\n\n def __len__(self):\n \"\"\"Returns the number of symbols in the dictionary\"\"\"\n return len(self.symbols)\n\n def __contains__(self, sym):\n return sym in self.indices\n\n def index(self, sym):\n \"\"\"Returns the index of the specified symbol\"\"\"\n assert isinstance(sym, str)\n if sym in self.alias_mapper:\n sym = self.alias_mapper[sym]\n if sym in self.indices:\n return self.indices[sym]\n return self.unk_index\n\n def unk_string(self, escape=False):\n \"\"\"Return unknown string, optionally escaped as: <>\"\"\"\n if escape:\n return \"<{}>\".format(self.unk_word)\n else:\n return self.unk_word\n\n def add_symbol(self, word, n=1, overwrite=False):\n \"\"\"Adds a word to the dictionary\"\"\"\n if word in self.alias_mapper:\n word = self.alias_mapper[word]\n if word in self.indices and not overwrite:\n idx = self.indices[word]\n self.count[idx] = self.count[idx] + n\n return idx\n else:\n idx = len(self.symbols)\n self.indices[word] = idx\n self.symbols.append(word)\n self.count.append(n)\n return idx\n\n def update(self, new_dict, word):\n \"\"\"Updates counts from new dictionary.\"\"\"\n if word in self.alias_mapper:\n word = self.alias_mapper[word]\n for word in new_dict.symbols:\n idx2 = new_dict.indices[word]\n if word in self.indices:\n idx = self.indices[word]\n self.count[idx] = self.count[idx] + new_dict.count[idx2]\n else:\n idx = len(self.symbols)\n self.indices[word] = idx\n self.symbols.append(word)\n self.count.append(new_dict.count[idx2])\n\n def pad_to_multiple_(self, padding_factor):\n \"\"\"Pad Dictionary size to be a multiple of *padding_factor*.\"\"\"\n if padding_factor > 1:\n i = 0\n while len(self) % padding_factor != 0:\n symbol = \"madeupword{:04d}\".format(i)\n self.add_symbol(symbol, n=0)\n i += 1\n\n def bos(self):\n \"\"\"Helper to get index of beginning-of-sentence symbol\"\"\"\n return self.bos_index\n\n def pad(self):\n \"\"\"Helper to get index of pad symbol\"\"\"\n return self.pad_index\n\n def eos(self):\n \"\"\"Helper to get index of end-of-sentence symbol\"\"\"\n return self.eos_index\n\n def unk(self):\n \"\"\"Helper to get index of unk symbol\"\"\"\n return self.unk_index\n\n @classmethod\n def load(cls, f):\n \"\"\"Loads the dictionary from a text file with the format:\n\n ```\n \n \n ...\n ```\n \"\"\"\n d = cls()\n d.add_from_file(f)\n return d\n\n def add_from_file(self, f):\n \"\"\"\n Loads a pre-existing dictionary from a text file and adds its symbols\n to this instance.\n \"\"\"\n if isinstance(f, str):\n try:\n # with open(PathManager.get_local_path(f), \"r\", encoding=\"utf-8\") as fd:\n with open(f, \"r\", encoding=\"utf-8\") as fd:\n self.add_from_file(fd)\n except FileNotFoundError as fnfe:\n raise fnfe\n except UnicodeError:\n raise Exception(\n \"Incorrect encoding detected in {}, please \"\n \"rebuild the dataset\".format(f)\n )\n return\n\n lines = f.readlines()\n indices_start_line = self._load_meta(lines)\n\n for line_idx, line in enumerate(lines[indices_start_line:]):\n try:\n splits = line.rstrip().rsplit(\" \", 1)\n line = splits[0]\n field = splits[1] if len(splits) > 1 else str(len(lines) - line_idx)\n if field == \"#fairseq:overwrite\":\n overwrite = True\n line, field = line.rsplit(\" \", 1)\n else:\n overwrite = False\n count = int(field)\n word = line\n if word in self and not overwrite:\n logger.info(\n \"Duplicate word found when loading Dictionary: '{}', index is {}.\".format(word, self.indices[word])\n )\n else:\n self.add_symbol(word, n=count, overwrite=overwrite)\n except ValueError:\n raise ValueError(\n \"Incorrect dictionary format, expected ' [flags]'\"\n )\n\n def _get_meta(self):\n return [], []\n\n def _load_meta(self, lines):\n return 0\n\n def save(self, f):\n \"\"\"Stores dictionary into a text file\"\"\"\n ex_keys, ex_vals = self._get_meta()\n self._save(\n f,\n zip(\n ex_keys + self.symbols[self.nspecial :],\n ex_vals + self.count[self.nspecial :],\n ),\n )\n\n def dummy_sentence(self, length):\n t = torch.Tensor(length).uniform_(self.nspecial + 1, len(self)).long()\n t[-1] = self.eos()\n return t\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:45:58.496887Z","iopub.execute_input":"2022-08-10T01:45:58.498471Z","iopub.status.idle":"2022-08-10T01:45:58.537122Z","shell.execute_reply.started":"2022-08-10T01:45:58.498431Z","shell.execute_reply":"2022-08-10T01:45:58.536058Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tokenization_cocolm.py\n\n# Copyright (c) Microsoft Corporation. \n# Licensed under the MIT license.\n# The script is largely adapted from the huggingface transformers library\n\nimport re\nimport os\nimport unicodedata\n\nfrom transformers.tokenization_utils import PreTrainedTokenizer\n# from cocolm.tokenization_utils import Dictionary\n\n\ndef _is_punctuation(char):\n \"\"\"Checks whether `chars` is a punctuation character.\"\"\"\n cp = ord(char)\n # We treat all non-letter/number ASCII as punctuation.\n # Characters such as \"^\", \"$\", and \"`\" are not in the Unicode\n # Punctuation class but we treat them as punctuation anyways, for\n # consistency.\n if ((cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or\n (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126)):\n return True\n cat = unicodedata.category(char)\n if cat.startswith(\"P\"):\n return True\n return False\n\n\ndef _truncate_seq_pair(tokens_a, tokens_b, max_length):\n \"\"\"Truncates a sequence pair in place to the maximum length.\"\"\"\n\n # This is a simple heuristic which will always truncate the longer sequence\n # one token at a time. This makes more sense than truncating an equal percent\n # of tokens from each, since if one sequence is very short then each token\n # that's truncated likely contains more information than a longer sequence.\n while True:\n total_length = len(tokens_a) + len(tokens_b)\n if total_length <= max_length:\n break\n if len(tokens_a) > len(tokens_b):\n tokens_a.pop()\n else:\n tokens_b.pop()\n\n\nclass SentencepiecePreTokenizer(object):\n\n def __init__(self):\n self.transl_table = dict( [ (ord(x), ord(y)) for x,y in zip( u\"‘’´“”—–-\", u\"'''\\\"\\\"---\") ] )\n\n def handle_single_quote(self, tokens):\n line = ' '.join(tokens)\n line = re.sub(r\"' ([smdSMDtT])\\b\", r\"'\\1\", line)\n line = re.sub(r\"' ll\\b\", \"'ll\", line)\n line = re.sub(r\"' re\\b\", \"'re\", line)\n line = re.sub(r\"' ve\\b\", \"'ve\", line)\n line = re.sub(r\"' LL\\b\", \"'LL \", line)\n line = re.sub(r\"' RE\\b\", \"'RE \", line)\n line = re.sub(r\"' VE\\b\", \"'VE \", line)\n return line.split()\n\n def split_on_cont_punc(self, tokens):\n new_tokens = []\n for token in tokens:\n if len(token) > 1:\n last_j = 0\n pre_is_punc = _is_punctuation(token[0])\n for j, ch in enumerate(token):\n is_punc = _is_punctuation(ch)\n if is_punc != pre_is_punc:\n new_tokens.append(token[last_j: j])\n last_j = j\n pre_is_punc = is_punc\n if last_j < len(token):\n new_tokens.append(token[last_j:])\n else:\n new_tokens.append(token)\n return new_tokens\n\n def split_pre_and_post_punc(self, tokens):\n def pre_punc(token):\n last_j = 0\n for j in range(1, len(token)):\n if not _is_punctuation(token[j]):\n last_j = j\n break\n return token[:last_j], token[last_j:]\n def post_punc(token):\n last_j = len(token)\n for j in range(len(token) - 2, -1, -1):\n is_punc = _is_punctuation(token[j])\n if not _is_punctuation(token[j]):\n last_j = j + 1\n break\n return token[:last_j], token[last_j:]\n new_tokens = []\n for token in tokens:\n if len(token) > 1 and _is_punctuation(token[0]):\n a, b = pre_punc(token)\n if a:\n new_tokens.append(a)\n if b:\n if _is_punctuation(b[-1]):\n c, d = post_punc(b)\n if c:\n new_tokens.append(c)\n if d:\n new_tokens.append(d)\n else:\n new_tokens.append(b)\n elif len(token) > 1 and _is_punctuation(token[-1]):\n a, b = post_punc(token)\n if a:\n new_tokens.append(a)\n if b:\n new_tokens.append(b)\n else:\n new_tokens.append(token)\n return new_tokens\n\n def tokenize(self, line):\n line = line.strip()\n line = line.replace(\"``\", '\"').replace(\"''\", '\"')\n line = line.translate(self.transl_table)\n tokens = line.split()\n tokens = self.split_pre_and_post_punc(tokens)\n tokens = self.handle_single_quote(tokens)\n return tokens\n\nCOCOLM_VOCAB_FILES_NAMES = {\"vocab_file\": \"sp.model\", \"dict_file\": \"dict.txt\"}\n\nCOCOLM_PRETRAINED_VOCAB_FILES_MAP = {\n \"vocab_file\": {\n \"cocolm-cased\": \"https://huggingface.co/microsoft/cocolm-base/resolve/main/sp.model\",\n },\n \"dict_file\": {\n \"cocolm-cased\": \"https://huggingface.co/microsoft/cocolm-base/resolve/main/dict.txt\"\n }\n}\n\nCOCOLM_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {\n \"cocolm-cased\": 512,\n}\n\nclass COCOLMTokenizer(PreTrainedTokenizer):\n vocab_files_names = COCOLM_VOCAB_FILES_NAMES\n pretrained_vocab_files_map = COCOLM_PRETRAINED_VOCAB_FILES_MAP\n max_model_input_sizes = COCOLM_PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES\n\n def __init__(self, vocab_file, dict_file, **kwargs):\n super(COCOLMTokenizer, self).__init__(**kwargs)\n print(f'vocab_file={vocab_file}; dict_file={dict_file}')\n if not os.path.exists(vocab_file):\n raise EnvironmentError(\"file {} not found\".format(vocab_file))\n try:\n import sentencepiece as spm\n\n self.sp = spm.SentencePieceProcessor()\n self.sp.Load(vocab_file)\n self.pre_tokenizer = SentencepiecePreTokenizer()\n self.dictionary = Dictionary.load(dict_file)\n except ImportError:\n raise ImportError('Please install sentencepiece with: pip install sentencepiece')\n self.dictionary.add_symbol('')\n\n @property\n def cls_token(self):\n return self.dictionary.alias_mapper[self.dictionary.bos_word]\n\n @property\n def sep_token(self):\n return self.dictionary.alias_mapper[self.dictionary.eos_word]\n\n @property\n def pad_token(self):\n return self.dictionary.alias_mapper[self.dictionary.pad_word]\n\n @property\n def unk_token(self):\n return self.dictionary.alias_mapper[self.dictionary.unk_word]\n\n @property\n def cls_token_id(self):\n return self.dictionary.bos_index\n\n @property\n def sep_token_id(self):\n return self.dictionary.eos_index\n\n @property\n def pad_token_id(self):\n return self.dictionary.pad_index\n\n @property\n def mask_token_id(self):\n return self.dictionary.index('')\n\n @property\n def unk_token_id(self):\n return self.dictionary.unk_index\n\n def encode_plus(self, text_a, text_b=None, add_special_tokens=True, max_length=512):\n tokens_a = self.tokenize(text_a)\n if text_b is not None:\n tokens_b = self.tokenize(text_b)\n _truncate_seq_pair(tokens_a, tokens_b, max_length - 4)\n else:\n if len(tokens_a) > max_length-2:\n tokens_a = tokens_a[:max_length-2]\n\n if add_special_tokens:\n tokens = [self.dictionary.bos_word] + tokens_a + [self.dictionary.eos_word]\n if text_b is not None:\n tokens += [self.dictionary.eos_word] + tokens_b + [self.dictionary.eos_word]\n else:\n tokens = tokens_a + tokens_b\n\n ids = self.convert_tokens_to_ids(tokens)\n return {\"input_ids\": ids}\n\n def encode(self, x: str, add_special_tokens=False) -> str:\n tokens = self.tokenize(x)\n return self.convert_tokens_to_ids(tokens)\n\n def decode(self, ids: list) -> str:\n x = \"\".join([self._convert_id_to_token(token_id) for token_id in ids])\n return x.replace(' ', '').replace('\\u2581', ' ').strip()\n\n def skip_space(self, tokens):\n new_tokens = []\n for i, token in enumerate(tokens):\n skip = False\n # skip single space, to reduce total length\n if token == '\\u2581':\n if i == len(tokens) - 1 or _is_punctuation(tokens[i + 1][0]):\n skip = True\n if not skip:\n new_tokens.append(token)\n return new_tokens\n\n def tokenize(self, x):\n x = ' '.join(self.pre_tokenizer.tokenize(x))\n tokens = self.sp.EncodeAsPieces(x)\n tokens = self.skip_space(tokens)\n return tokens\n\n def convert_tokens_to_ids(self, tokens: list):\n ret = []\n if isinstance(tokens, str):\n return self.dictionary.index(tokens)\n for token in tokens:\n ret.append(self.dictionary.index(token))\n return ret\n \n def _convert_id_to_token(self, index):\n \"\"\" Converts a token (str) in an id using the vocab. \"\"\"\n token = self.dictionary[index]\n return token\n\n def convert_tokens_to_string(self, tokens: list):\n x = \" \".join(tokens)\n return x.replace(' ', '').replace('\\u2581', ' ').strip()\n\n def is_beginning_of_word(self, x: str) -> bool:\n if x in [\"\", \"\", \"\", \"\", \"[CLS]\", \"[PAD]\", \"[SEP]\", \"[UNK]\"]:\n # special elements are always considered beginnings\n # HACK: this logic is already present in fairseq/tasks/masked_lm.py\n # but these special tokens are also contained in the sentencepiece\n # vocabulary which causes duplicate special tokens. This hack makes\n # sure that they are all taken into account.\n return True\n return x.startswith(\"\\u2581\")\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:45:58.539102Z","iopub.execute_input":"2022-08-10T01:45:58.539480Z","iopub.status.idle":"2022-08-10T01:45:58.579372Z","shell.execute_reply.started":"2022-08-10T01:45:58.539445Z","shell.execute_reply":"2022-08-10T01:45:58.578344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n# tokenizer_cocolm = COCOLMTokenizer.from_pretrained(\"microsoft/cocolm-base\")\ntokenizer_cocolm = COCOLMTokenizer.from_pretrained(\"../input/cocobase/cocolm-base/\")","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:46:02.911011Z","iopub.execute_input":"2022-08-10T01:46:02.911689Z","iopub.status.idle":"2022-08-10T01:46:03.155003Z","shell.execute_reply.started":"2022-08-10T01:46:02.911650Z","shell.execute_reply":"2022-08-10T01:46:03.154039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n\nCURRENT_MODEL_OUTPUT_PATH = '../input/feedback-cocolm-base-2/'\n# CURRENT_MODEL_OUTPUT_PATH = '../input/cocolm-base-2-layernorm/'\n\nMODEL_PATHS = [\n CURRENT_MODEL_OUTPUT_PATH + 'Loss-Fold-0.bin',\n CURRENT_MODEL_OUTPUT_PATH + 'Loss-Fold-1.bin',\n CURRENT_MODEL_OUTPUT_PATH + 'Loss-Fold-2.bin'\n]\n\n# 设置CFG参数\n\nCONFIG = dict(\n seed = 666,\n# model_name = '../input/deberta-v3-base',\n test_batch_size = 6,\n max_length = 512,\n num_classes = 3,\n device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n)\n\nCONFIG[\"tokenizer\"] = tokenizer_cocolm # AutoTokenizer.from_pretrained(CONFIG['model_name'])\n\ntest_path = \"../input/feedback-prize-effectiveness/test\"\n\ndef get_essay(essay_id):\n essay_path = os.path.join(test_path, f\"{essay_id}.txt\")\n essay_text = open(essay_path, 'r').read()\n return essay_text\n\ndf = pd.read_csv(\"../input/feedback-prize-effectiveness/test.csv\")\ndf['essay_text'] = df['essay_id'].apply(get_essay)\ndf.head()\n\nwith open(\"../input/feedback-cocolm-base-2/le.pkl\", \"rb\") as fp:\n encoder = joblib.load(fp)\n\nprint(encoder.classes_)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:46:26.819710Z","iopub.execute_input":"2022-08-10T01:46:26.820074Z","iopub.status.idle":"2022-08-10T01:46:26.846198Z","shell.execute_reply.started":"2022-08-10T01:46:26.820043Z","shell.execute_reply":"2022-08-10T01:46:26.845238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nclass FeedBackDataset(Dataset):\n def __init__(self, df, tokenizer, max_length):\n self.df = df\n self.max_len = max_length\n self.tokenizer = tokenizer\n self.discourse = df['discourse_text'].values\n self.discourse_type = df['discourse_type'].values\n self.essay = df['essay_text'].values\n\n def __len__(self):\n return len(self.df)\n\n def __getitem__(self, index):\n discourse = self.discourse[index]\n essay = self.essay[index]\n discourse_type = self.discourse_type[index]\n# text = discourse + \" \" + self.tokenizer.sep_token + \" \" + essay\n inputs = self.tokenizer.encode_plus(\n discourse_type + \" \" + discourse,\n essay,\n add_special_tokens=True,\n max_length=self.max_len\n )\n\n input_ids = inputs[\"input_ids\"]\n \n mask_padding_with_zero = True\n pad_token_id = self.tokenizer.convert_tokens_to_ids([self.tokenizer.pad_token])[0]\n \n # The mask has 1 for real tokens and 0 for padding tokens. Only real\n # tokens are attended to.\n attention_mask = [1 if mask_padding_with_zero else 0] * len(input_ids)\n\n # Zero-pad up to the sequence length.\n padding_length = self.max_len - len(input_ids)\n \n input_ids = input_ids + ([pad_token_id] * padding_length)\n attention_mask = attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)\n\n return {\n 'ids': torch.tensor(input_ids, dtype=torch.long),\n 'mask': torch.tensor(attention_mask, dtype=torch.long)\n }\n\ntest_dataset = FeedBackDataset(df, CONFIG['tokenizer'], max_length=CONFIG['max_length'])\ntest_loader = DataLoader(test_dataset, batch_size=CONFIG['test_batch_size'],\n num_workers=2, shuffle=False, pin_memory=True)\n\n\nclass MeanPooling(nn.Module):\n def __init__(self):\n super(MeanPooling, self).__init__()\n\n def forward(self, last_hidden_state, attention_mask):\n input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n sum_mask = input_mask_expanded.sum(1)\n sum_mask = torch.clamp(sum_mask, min=1e-9)\n mean_embeddings = sum_embeddings / sum_mask\n return mean_embeddings\n\n\nclass FeedBackModel(nn.Module):\n def __init__(self):\n super(FeedBackModel, self).__init__()\n self.model = model_cocolm\n self.config = config_cocolm\n self.drop = nn.Dropout(p=0.2)\n# self.ln = nn.LayerNorm(self.config.hidden_size)\n self.pooler = MeanPooling()\n self.fc = nn.Linear(self.config.hidden_size, CONFIG['num_classes'])\n\n def forward(self, ids, mask):\n out = self.model(input_ids=ids, attention_mask=mask)\n# print(f'out = {out}')\n# out = self.pooler(out.last_hidden_state, mask)\n# out = self.ln(out[0][:, 0, :])\n out = self.drop(out[0][:, 0, :])\n outputs = self.fc(out)\n return outputs\n\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:47:44.239722Z","iopub.execute_input":"2022-08-10T01:47:44.240114Z","iopub.status.idle":"2022-08-10T01:47:44.278856Z","shell.execute_reply.started":"2022-08-10T01:47:44.240080Z","shell.execute_reply":"2022-08-10T01:47:44.277665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n@torch.no_grad()\ndef valid_fn(model, dataloader, device):\n model.eval()\n\n dataset_size = 0\n running_loss = 0.0\n\n preds = []\n\n bar = tqdm(enumerate(dataloader), total=len(dataloader))\n for step, data in bar:\n ids = data['ids'].to(device, dtype=torch.long)\n mask = data['mask'].to(device, dtype=torch.long)\n\n outputs = model(ids, mask)\n outputs = F.softmax(outputs, dim=1)\n preds.append(outputs.cpu().detach().numpy())\n\n preds = np.concatenate(preds)\n gc.collect()\n\n return preds\n\n\ndef inference(model_paths, dataloader, device):\n final_preds = []\n for i, path in enumerate(model_paths):\n model = FeedBackModel()\n model.to(CONFIG['device'])\n model.load_state_dict(torch.load(path, map_location=CONFIG['device']))\n\n print(f\"Load model and predictions {i + 1}\")\n preds = valid_fn(model, dataloader, device)\n final_preds.append(preds)\n\n final_preds = np.array(final_preds)\n# final_preds = np.mean(final_preds, axis=0)\n return final_preds\n\nmodel_preds = inference(MODEL_PATHS, test_loader, CONFIG['device'])\n\n# sample_submission = pd.read_csv(\"../input/feedback-prize-effectiveness/sample_submission.csv\")\n# print(sample_submission.head())\n\n# sample_submission['Adequate'] = np.mean(model_preds[:, :, 0], axis=0)\n# sample_submission['Effective'] = np.mean(model_preds[:, :, 1], axis=0)\n# sample_submission['Ineffective'] = np.mean(model_preds[:, :, 2], axis=0)\n# print(sample_submission.head())\n\ncocolm_base_ineffective = []\ncocolm_base_effective = []\ncocolm_base_adequate = []\n\nfor x in model_preds:\n cocolm_base_ineffective.append(x[:, 2]) \n cocolm_base_effective.append(x[:, 1])\n cocolm_base_adequate.append(x[:, 0])\n\n# list -> dataframe\ncocolm_base_ineffective = pd.DataFrame(cocolm_base_ineffective).T\ncocolm_base_adequate = pd.DataFrame(cocolm_base_adequate).T\ncocolm_base_effective = pd.DataFrame(cocolm_base_effective).T\n\n# display(cocolm_base_ineffective)\n# display(cocolm_base_adequate)\n# display(cocolm_base_effective)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:47:46.921164Z","iopub.execute_input":"2022-08-10T01:47:46.921856Z","iopub.status.idle":"2022-08-10T01:48:11.640340Z","shell.execute_reply.started":"2022-08-10T01:47:46.921821Z","shell.execute_reply":"2022-08-10T01:48:11.639347Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 二、deberta-v3-base","metadata":{}},{"cell_type":"code","source":"import os\nimport gc\nimport torch\nimport pickle\nimport codecs\nimport gensim\nimport numpy as np\nimport pandas as pd\nimport pickle as pkl\nimport torch.nn as nn\nfrom tqdm import tqdm\nimport seaborn as sns\nimport torch.nn as nn\nimport lightgbm as lgb\nfrom scipy import sparse\nfrom typing import Tuple\nimport torch.nn.functional as F\nfrom sklearn.metrics import log_loss\nfrom text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nfrom transformers import AutoTokenizer\nfrom sklearn.preprocessing import OneHotEncoder\nfrom torch.utils.data import Dataset, DataLoader\nfrom sklearn.model_selection import StratifiedGroupKFold\nfrom transformers import AutoModel, AutoTokenizer, AutoConfig\nimport warnings; warnings.simplefilter('ignore')\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:51:14.880713Z","iopub.execute_input":"2022-08-10T01:51:14.881300Z","iopub.status.idle":"2022-08-10T01:51:17.512642Z","shell.execute_reply.started":"2022-08-10T01:51:14.881264Z","shell.execute_reply":"2022-08-10T01:51:17.511506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = '../input/feedback-prize-effectiveness/'\n\nclass CFG:\n CVs = []\n seed = 42\n lr = 3e-5\n epochs = 3\n n_fold = 5\n apex = True\n fast = True\n AMP = False\n n_splits = 5\n train = True\n wandb = False\n max_len = 512\n dropout = 0.1\n min_lr = 1e-6\n batch_size = 8\n freezing = True\n print_freq = 50\n target_size = 3\n num_workers = 0\n num_cycles = 0.5\n n_accumulate = 1\n scheduler = 'cosine'\n weigth_decay = 0.01\n num_warmup_steps = 0\n trn_fold = [0, 1, 2, 3, 4]\n gradient_checkpointing = True\n model = '../input/deberta-v3-base/deberta-v3-base'\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:48:42.724182Z","iopub.execute_input":"2022-08-10T01:48:42.724899Z","iopub.status.idle":"2022-08-10T01:48:42.732029Z","shell.execute_reply.started":"2022-08-10T01:48:42.724862Z","shell.execute_reply":"2022-08-10T01:48:42.731056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def criterion(outputs, labels):\n return nn.CrossEntropyLoss()(outputs, labels)\n\ndef softmax(z):\n assert len(z.shape) == 2\n s = np.max(z, axis=1)\n s = s[:, np.newaxis]\n e_x = np.exp(z - s)\n div = np.sum(e_x, axis=1)\n div = div[:, np.newaxis]\n return e_x / div\n\ndef freeze(module):\n for parameter in module.parameters():\n parameter.requires_grad = False\n \ndef get_freezed_parameters(module):\n freezed_parameters = []\n for name, parameter in module.named_parameters():\n if not parameter.requires_grad:\n freezed_parameters.append(name)\n return freezed_parameters\n\ndef get_essay(essay_id, is_train=True):\n parent_path = INPUT_DIR + 'train' if is_train else INPUT_DIR + 'test'\n essay_path = os.path.join(parent_path, f\"{essay_id}.txt\")\n essay_text = open(essay_path, 'r').read()\n return essay_text","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:49:06.073278Z","iopub.execute_input":"2022-08-10T01:49:06.073951Z","iopub.status.idle":"2022-08-10T01:49:06.084534Z","shell.execute_reply.started":"2022-08-10T01:49:06.073911Z","shell.execute_reply":"2022-08-10T01:49:06.083266Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Testing Data\ntest = pd.read_csv(INPUT_DIR + 'test.csv')\ntest['essay_text'] = test['essay_id'].apply(lambda x: get_essay(x, is_train=False))","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:49:26.778822Z","iopub.execute_input":"2022-08-10T01:49:26.779169Z","iopub.status.idle":"2022-08-10T01:49:26.796405Z","shell.execute_reply.started":"2022-08-10T01:49:26.779140Z","shell.execute_reply":"2022-08-10T01:49:26.795526Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if CFG.fast: tokenizer = AutoTokenizer.from_pretrained(CFG.model, use_fast=True)\nelse: tokenizer = AutoTokenizer.from_pretrained(CFG.model)\nCFG.tokenizer = tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:49:43.034708Z","iopub.execute_input":"2022-08-10T01:49:43.035106Z","iopub.status.idle":"2022-08-10T01:49:43.733392Z","shell.execute_reply.started":"2022-08-10T01:49:43.035072Z","shell.execute_reply":"2022-08-10T01:49:43.732267Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]: return error.object[error.start : error.end].encode(\"utf-8\"), error.end\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]: return error.object[error.start : error.end].decode(\"cp1252\"), error.end\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n text = (text.encode(\"raw_unicode_escape\").decode(\"utf-8\", errors = \"replace_decoding_with_cp1252\").encode(\"cp1252\", errors = \"replace_encoding_with_utf8\").decode(\"utf-8\", errors = \"replace_decoding_with_cp1252\"))\n text = unidecode(text)\n return text","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:51:33.322697Z","iopub.execute_input":"2022-08-10T01:51:33.323055Z","iopub.status.idle":"2022-08-10T01:51:33.331231Z","shell.execute_reply.started":"2022-08-10T01:51:33.323026Z","shell.execute_reply":"2022-08-10T01:51:33.329978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['discourse_text'] = test['discourse_text'].apply(lambda x : resolve_encodings_and_normalize(x))\ntest['essay_text'] = test['essay_text'].apply(lambda x : resolve_encodings_and_normalize(x))\ntest['text'] = test['discourse_type'] + ' ' + test['discourse_text'] + '[SEP]' + test['essay_text']\ntest['label'] = np.nan","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:52:08.398466Z","iopub.execute_input":"2022-08-10T01:52:08.398889Z","iopub.status.idle":"2022-08-10T01:52:08.415629Z","shell.execute_reply.started":"2022-08-10T01:52:08.398852Z","shell.execute_reply":"2022-08-10T01:52:08.414454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(Dataset):\n def __init__(self, cfg, df):\n self.cfg = cfg\n self.text = df['text'].values\n def __len__(self): return len(self.text)\n def __getitem__(self, item):\n inputs = self.cfg.tokenizer.encode_plus(self.text[item], truncation = True, add_special_tokens = True, max_length = self.cfg.max_len)\n samples = {'input_ids': inputs['input_ids'], 'attention_mask': inputs['attention_mask'], }\n if 'token_type_ids' in inputs: samples['token_type_ids'] = inputs['token_type_ids']\n return samples\n\nclass Collate:\n def __init__(self, tokenizer, isTrain=True):\n self.isTrain = isTrain\n self.tokenizer = tokenizer\n\n def __call__(self, batch):\n output = dict()\n output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n if self.isTrain: output[\"target\"] = [sample[\"target\"] for sample in batch]\n batch_max = max([len(ids) for ids in output[\"input_ids\"]])\n if self.tokenizer.padding_side == \"right\":\n output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n else:\n output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n if self.isTrain: output[\"target\"] = torch.tensor(output[\"target\"], dtype=torch.long)\n return output","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:52:27.782175Z","iopub.execute_input":"2022-08-10T01:52:27.782676Z","iopub.status.idle":"2022-08-10T01:52:27.802434Z","shell.execute_reply.started":"2022-08-10T01:52:27.782636Z","shell.execute_reply":"2022-08-10T01:52:27.801154Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MeanPooling(nn.Module):\n def __init__(self):\n super(MeanPooling, self).__init__()\n \n def forward(self, last_hidden_state, attention_mask):\n input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n sum_mask = input_mask_expanded.sum(1)\n sum_mask = torch.clamp(sum_mask, min=1e-9) #\n mean_embeddings = sum_embeddings / sum_mask\n return mean_embeddings\n\ndef inference_fn(test_loader, model, device):\n preds = []\n model.eval()\n model.to(device)\n tk0 = tqdm(test_loader, total=len(test_loader))\n for data in tk0:\n ids = data['input_ids'].to(device, dtype = torch.long)\n mask = data['attention_mask'].to(device, dtype = torch.long)\n with torch.no_grad():\n y_preds = model(ids, mask)\n y_preds = softmax(y_preds.to('cpu').numpy())\n preds.append(y_preds)\n predictions = np.concatenate(preds)\n return predictions","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:52:40.937323Z","iopub.execute_input":"2022-08-10T01:52:40.937872Z","iopub.status.idle":"2022-08-10T01:52:40.950528Z","shell.execute_reply.started":"2022-08-10T01:52:40.937830Z","shell.execute_reply":"2022-08-10T01:52:40.949519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class FeedBackModel(nn.Module):\n def __init__(self, model_name):\n super(FeedBackModel, self).__init__()\n self.model = AutoModel.from_pretrained(model_name)\n if CFG.gradient_checkpointing: (self.model).gradient_checkpointing_enable()\n if CFG.freezing:\n freeze((self.model).embeddings)\n freeze((self.model).encoder.layer[:2])\n CFG.after_freezed_parameters = filter(lambda parameter: parameter.requires_grad, (self.model).parameters())\n self.config = AutoConfig.from_pretrained(model_name)\n self.drop = nn.Dropout(p=CFG.dropout)\n self.pooler = MeanPooling()\n self.fc = nn.Linear(self.config.hidden_size, CFG.target_size)\n \n def forward(self, ids, mask):\n out = self.model(input_ids = ids, attention_mask = mask, output_hidden_states = False)\n out = self.pooler(out.last_hidden_state, mask)\n out = self.drop(out)\n outputs = self.fc(out)\n return outputs","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:52:54.763388Z","iopub.execute_input":"2022-08-10T01:52:54.763867Z","iopub.status.idle":"2022-08-10T01:52:54.779110Z","shell.execute_reply.started":"2022-08-10T01:52:54.763825Z","shell.execute_reply":"2022-08-10T01:52:54.778029Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"testDataset = TestDataset(CFG, test)\ntest_loader = DataLoader(\n testDataset,\n shuffle = False,\n drop_last = False,\n pin_memory = True,\n batch_size = CFG.batch_size,\n num_workers = CFG.num_workers,\n collate_fn = Collate(CFG.tokenizer, isTrain = False)\n )\n\ndeberta_predictions = []\nfor i in CFG.trn_fold:\n model = FeedBackModel(CFG.model)\n model.load_state_dict(torch.load('../input/dbv3basemodels202279/models-deberta-v3-base-deberta-v3-base_fold' + str(i) +'_best.pth'))\n prediction = inference_fn(test_loader, model, device)\n deberta_predictions.append(prediction)\n torch.cuda.empty_cache()\n gc.collect()\n\ndeberta_base_adequate = []\ndeberta_base_effective = []\ndeberta_base_ineffective = []\n\nfor x in deberta_predictions:\n deberta_base_ineffective.append(x[:, 0])\n deberta_base_adequate.append(x[:, 1])\n deberta_base_effective.append(x[:, 2])\n\ndeberta_base_ineffective = pd.DataFrame(deberta_base_ineffective).T\ndeberta_base_adequate = pd.DataFrame(deberta_base_adequate).T\ndeberta_base_effective = pd.DataFrame(deberta_base_effective).T\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:55:58.593377Z","iopub.execute_input":"2022-08-10T01:55:58.594320Z","iopub.status.idle":"2022-08-10T01:56:16.250491Z","shell.execute_reply.started":"2022-08-10T01:55:58.594269Z","shell.execute_reply":"2022-08-10T01:56:16.249586Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 三、deberta-large\n\nhttps://www.kaggle.com/code/brandonhu0215/feedback-deberta-large-lb0-619","metadata":{}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\n\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport shutil\n\nfrom torch.utils.data import DataLoader, Dataset\nimport datasets, transformers\nfrom transformers import TrainingArguments, Trainer\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer, DataCollatorWithPadding\n\nimport os\nimport gc\nimport re\nimport ast\nimport sys\nimport copy\nimport json\nimport time\nimport math\nimport shutil\nimport string\nimport pickle\nimport random\nimport joblib\nimport itertools\nfrom pathlib import Path\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom sklearn.metrics import f1_score, log_loss\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\n\nimport torch\nprint(f\"torch.__version__: {torch.__version__}\")\nimport torch.nn as nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\n\n# os.system('pip uninstall -y transformers')\n# os.system('pip uninstall -y tokenizers')\n# os.system('python -m pip install --no-index --find-links=../input/pppm-pip-wheels transformers')\n# os.system('python -m pip install --no-index --find-links=../input/pppm-pip-wheels tokenizers')\nimport tokenizers\nimport transformers\nprint(f\"tokenizers.__version__: {tokenizers.__version__}\")\nprint(f\"transformers.__version__: {transformers.__version__}\")\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n%env TOKENIZERS_PARALLELISM=true\n\ngc.collect()\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:57:55.451386Z","iopub.execute_input":"2022-08-10T01:57:55.452389Z","iopub.status.idle":"2022-08-10T01:58:00.306323Z","shell.execute_reply.started":"2022-08-10T01:57:55.452352Z","shell.execute_reply":"2022-08-10T01:58:00.305319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class CFG:\n num_workers=1\n path=\"../input/feedback-deberta-large-051/\"\n config_path=path+'config.pth'\n model=\"microsoft/deberta-large\"\n batch_size=16\n fc_dropout=0.2\n target_size=3\n max_len=512\n seed=42\n n_fold=4\n trn_fold=[i for i in range(n_fold)]\n gradient_checkpoint=False","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:00.308416Z","iopub.execute_input":"2022-08-10T01:58:00.309121Z","iopub.status.idle":"2022-08-10T01:58:00.315355Z","shell.execute_reply.started":"2022-08-10T01:58:00.309083Z","shell.execute_reply":"2022-08-10T01:58:00.314241Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\n\ndef get_essay(essay_id, is_train=True):\n parent_path = INPUT_DIR + 'train' if is_train else INPUT_DIR + 'test'\n essay_path = os.path.join(parent_path, f\"{essay_id}.txt\")\n essay_text = open(essay_path, 'r').read()\n return essay_text\n\ndef softmax(z):\n assert len(z.shape) == 2\n s = np.max(z, axis=1)\n s = s[:, np.newaxis] # necessary step to do broadcasting\n e_x = np.exp(z - s)\n div = np.sum(e_x, axis=1)\n div = div[:, np.newaxis] # dito\n return e_x / div\n\ndef get_score(y_true, y_pred):\n y_pred = softmax(y_pred)\n score = log_loss(y_true, y_pred)\n return round(score, 5)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:00.316985Z","iopub.execute_input":"2022-08-10T01:58:00.317724Z","iopub.status.idle":"2022-08-10T01:58:00.329458Z","shell.execute_reply.started":"2022-08-10T01:58:00.317689Z","shell.execute_reply":"2022-08-10T01:58:00.328298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"INPUT_DIR = \"../input/feedback-prize-effectiveness/\"\ntest = pd.read_csv(os.path.join(INPUT_DIR, 'test.csv'))\nsubmission = pd.read_csv(os.path.join(INPUT_DIR, 'sample_submission.csv'))\ntest['essay_text'] = test['essay_id'].apply(lambda x: get_essay(x, is_train=False))\n","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:00.608160Z","iopub.execute_input":"2022-08-10T01:58:00.608661Z","iopub.status.idle":"2022-08-10T01:58:00.629709Z","shell.execute_reply.started":"2022-08-10T01:58:00.608633Z","shell.execute_reply":"2022-08-10T01:58:00.628864Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# tokenizer\n# ====================================================\ntokenizer = AutoTokenizer.from_pretrained(CFG.path + 'tokenizer')\nCFG.tokenizer = tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:02.175581Z","iopub.execute_input":"2022-08-10T01:58:02.176874Z","iopub.status.idle":"2022-08-10T01:58:02.369743Z","shell.execute_reply.started":"2022-08-10T01:58:02.176825Z","shell.execute_reply":"2022-08-10T01:58:02.368592Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\ndef replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n# Register the encoding and decoding error handlers for `utf-8` and `cp1252`.\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n \"\"\"Resolve the encoding problems and normalize the abnormal characters.\"\"\"\n text = (\n text.encode(\"raw_unicode_escape\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n )\n text = unidecode(text)\n return text","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:03.828919Z","iopub.execute_input":"2022-08-10T01:58:03.829346Z","iopub.status.idle":"2022-08-10T01:58:03.839395Z","shell.execute_reply.started":"2022-08-10T01:58:03.829304Z","shell.execute_reply":"2022-08-10T01:58:03.837955Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test['discourse_text'] = test['discourse_text'].apply(lambda x : resolve_encodings_and_normalize(x))\ntest['essay_text'] = test['essay_text'].apply(lambda x : resolve_encodings_and_normalize(x))","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:06.134319Z","iopub.execute_input":"2022-08-10T01:58:06.135753Z","iopub.status.idle":"2022-08-10T01:58:06.149355Z","shell.execute_reply.started":"2022-08-10T01:58:06.135710Z","shell.execute_reply":"2022-08-10T01:58:06.148335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEP = tokenizer.sep_token\ntest['text'] = test['discourse_type'] + ' ' + test['discourse_text'] # + SEP + test['essay_text']\ntest['label'] = np.nan\ndisplay(test.head())","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:07.520460Z","iopub.execute_input":"2022-08-10T01:58:07.521559Z","iopub.status.idle":"2022-08-10T01:58:07.546250Z","shell.execute_reply.started":"2022-08-10T01:58:07.521514Z","shell.execute_reply":"2022-08-10T01:58:07.545141Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class TestDataset(Dataset):\n def __init__(self, cfg, df):\n self.cfg = cfg\n self.text = df['text'].values\n self.text_essay = df['essay_text'].values \n\n def __len__(self):\n return len(self.text)\n\n def __getitem__(self, item):\n inputs = self.cfg.tokenizer.encode_plus(\n self.text[item],\n self.text_essay[item],\n truncation=True,\n add_special_tokens=True,\n max_length=self.cfg.max_len\n )\n samples = {\n 'input_ids': inputs['input_ids'],\n 'attention_mask': inputs['attention_mask'],\n }\n\n if 'token_type_ids' in inputs:\n samples['token_type_ids'] = inputs['token_type_ids']\n \n return samples","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:09.809905Z","iopub.execute_input":"2022-08-10T01:58:09.811207Z","iopub.status.idle":"2022-08-10T01:58:09.821093Z","shell.execute_reply.started":"2022-08-10T01:58:09.811075Z","shell.execute_reply":"2022-08-10T01:58:09.819640Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class Collate:\n def __init__(self, tokenizer, isTrain=True):\n self.tokenizer = tokenizer\n self.isTrain = isTrain\n # self.args = args\n\n def __call__(self, batch):\n output = dict()\n output[\"input_ids\"] = [sample[\"input_ids\"] for sample in batch]\n output[\"attention_mask\"] = [sample[\"attention_mask\"] for sample in batch]\n if self.isTrain:\n output[\"target\"] = [sample[\"target\"] for sample in batch]\n\n # calculate max token length of this batch\n batch_max = max([len(ids) for ids in output[\"input_ids\"]])\n\n # add padding\n if self.tokenizer.padding_side == \"right\":\n output[\"input_ids\"] = [s + (batch_max - len(s)) * [self.tokenizer.pad_token_id] for s in output[\"input_ids\"]]\n output[\"attention_mask\"] = [s + (batch_max - len(s)) * [0] for s in output[\"attention_mask\"]]\n else:\n output[\"input_ids\"] = [(batch_max - len(s)) * [self.tokenizer.pad_token_id] + s for s in output[\"input_ids\"]]\n output[\"attention_mask\"] = [(batch_max - len(s)) * [0] + s for s in output[\"attention_mask\"]]\n\n # convert to tensors\n output[\"input_ids\"] = torch.tensor(output[\"input_ids\"], dtype=torch.long)\n output[\"attention_mask\"] = torch.tensor(output[\"attention_mask\"], dtype=torch.long)\n if self.isTrain:\n output[\"target\"] = torch.tensor(output[\"target\"], dtype=torch.long)\n\n return output\n\ncollate_fn = Collate(CFG.tokenizer, isTrain=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:11.031729Z","iopub.execute_input":"2022-08-10T01:58:11.032491Z","iopub.status.idle":"2022-08-10T01:58:11.045491Z","shell.execute_reply.started":"2022-08-10T01:58:11.032452Z","shell.execute_reply":"2022-08-10T01:58:11.043913Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Model\n# ====================================================\nfrom torch.cuda.amp import autocast\nclass CustomModel(nn.Module):\n def __init__(self, cfg, config_path=None, pretrained=False):\n super().__init__()\n self.cfg = cfg\n if config_path is None:\n self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n else:\n self.config = torch.load(config_path)\n \n if pretrained:\n self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n else:\n self.model = AutoModel.from_config(self.config)\n \n # gradient checkpointing\n if self.cfg.gradient_checkpoint:\n self.model.gradient_checkpointing_enable()\n print(f\"Gradient Checkpointing: {self.model.is_gradient_checkpointing}\")\n \n \n # self.pooler = MeanPooling()\n \n self.bilstm = nn.LSTM(self.config.hidden_size, (self.config.hidden_size) // 2, num_layers=2, \n dropout=self.config.hidden_dropout_prob, batch_first=True,\n bidirectional=True)\n \n self.dropout = nn.Dropout(0.2)\n self.dropout1 = nn.Dropout(0.1)\n self.dropout2 = nn.Dropout(0.2)\n self.dropout3 = nn.Dropout(0.3)\n self.dropout4 = nn.Dropout(0.4)\n self.dropout5 = nn.Dropout(0.5)\n \n self.output = nn.Sequential(\n nn.Linear(self.config.hidden_size, self.cfg.target_size)\n # nn.Linear(256, self.cfg.target_size)\n )\n \n \n\n def loss(self, outputs, targets):\n loss_fct = nn.CrossEntropyLoss()\n loss = loss_fct(outputs, targets)\n return loss\n \n def monitor_metrics(self, outputs, targets):\n device = targets.get_device()\n # print(outputs)\n # print(targets)\n mll = log_loss(\n targets.cpu().detach().numpy(),\n softmax(outputs.cpu().detach().numpy()),\n labels=[0, 1, 2],\n )\n return mll\n \n def _init_weights(self, module):\n if isinstance(module, nn.Linear):\n module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n if module.bias is not None:\n module.bias.data.zero_()\n elif isinstance(module, nn.Embedding):\n module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n if module.padding_idx is not None:\n module.weight.data[module.padding_idx].zero_()\n elif isinstance(module, nn.LayerNorm):\n module.bias.data.zero_()\n module.weight.data.fill_(1.0)\n\n def forward(self, ids, mask, token_type_ids=None, targets=None):\n if token_type_ids:\n transformer_out = self.model(ids, mask, token_type_ids)\n else:\n transformer_out = self.model(ids, mask)\n \n # LSTM/GRU header\n# all_hidden_states = torch.stack(transformer_out[1])\n# sequence_output = self.pooler(all_hidden_states)\n \n # simple CLS\n sequence_output = transformer_out[0][:, 0, :]\n\n \n # Main task\n logits1 = self.output(self.dropout1(sequence_output))\n logits2 = self.output(self.dropout2(sequence_output))\n logits3 = self.output(self.dropout3(sequence_output))\n logits4 = self.output(self.dropout4(sequence_output))\n logits5 = self.output(self.dropout5(sequence_output))\n logits = (logits1 + logits2 + logits3 + logits4 + logits5) / 5\n\n if targets is not None:\n metric = self.monitor_metrics(logits, targets)\n return logits, metric\n \n return logits, 0.","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:12.287738Z","iopub.execute_input":"2022-08-10T01:58:12.288086Z","iopub.status.idle":"2022-08-10T01:58:12.306053Z","shell.execute_reply.started":"2022-08-10T01:58:12.288056Z","shell.execute_reply":"2022-08-10T01:58:12.304886Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# inference\n# ====================================================\ndef inference_fn(test_loader, model, device):\n preds = []\n model.eval()\n model.to(device)\n tk0 = tqdm(test_loader, total=len(test_loader))\n for data in tk0:\n ids = data['input_ids'].to(device, dtype = torch.long)\n mask = data['attention_mask'].to(device, dtype = torch.long)\n with torch.no_grad():\n y_preds, _ = model(ids, mask)\n y_preds = softmax(y_preds.to('cpu').numpy())\n preds.append(y_preds)\n predictions = np.concatenate(preds)\n return predictions","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:13.808183Z","iopub.execute_input":"2022-08-10T01:58:13.809095Z","iopub.status.idle":"2022-08-10T01:58:13.817571Z","shell.execute_reply.started":"2022-08-10T01:58:13.809057Z","shell.execute_reply":"2022-08-10T01:58:13.816070Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deberta_predictions = []\ntest_dataset = TestDataset(CFG, test)\ntest_loader = DataLoader(test_dataset,\n batch_size=CFG.batch_size,\n shuffle=False,\n collate_fn=collate_fn,\n num_workers=CFG.num_workers, pin_memory=True, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:23.011775Z","iopub.execute_input":"2022-08-10T01:58:23.012140Z","iopub.status.idle":"2022-08-10T01:58:23.019268Z","shell.execute_reply.started":"2022-08-10T01:58:23.012108Z","shell.execute_reply":"2022-08-10T01:58:23.017532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"deberta_predictions = []\nfor fold in CFG.trn_fold:\n print(\"Fold {}\".format(fold))\n\n model = CustomModel(CFG, config_path=CFG.config_path, pretrained=False)\n state = torch.load(CFG.path+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\",\n map_location=torch.device('cpu'))\n model.load_state_dict(state['model'])\n prediction = inference_fn(test_loader, model, device)\n deberta_predictions.append(prediction)\n del model, state, prediction; gc.collect()\n torch.cuda.empty_cache()","metadata":{"execution":{"iopub.status.busy":"2022-08-10T01:58:28.737694Z","iopub.execute_input":"2022-08-10T01:58:28.738055Z","iopub.status.idle":"2022-08-10T02:00:33.456461Z","shell.execute_reply.started":"2022-08-10T01:58:28.738025Z","shell.execute_reply":"2022-08-10T02:00:33.455076Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# predictions = np.mean(deberta_predictions, axis=0)\n# submission['Ineffective'] = predictions[:, 0]\n# submission['Adequate'] = predictions[:, 1]\n# submission['Effective'] = predictions[:, 2]\n\n\ndeberta_large_ineffective = []\ndeberta_large_effective = []\ndeberta_large_adequate = []\n\n# ['Adequate' 'Effective' 'Ineffective']\nfor x in deberta_predictions:\n deberta_large_ineffective.append(x[:, 0])\n deberta_large_effective.append(x[:, 2])\n deberta_large_adequate.append(x[:, 1])\n\n# list -> dataframe\ndeberta_large_ineffective = pd.DataFrame(deberta_large_ineffective).T\ndeberta_large_adequate = pd.DataFrame(deberta_large_adequate).T\ndeberta_large_effective = pd.DataFrame(deberta_large_effective).T\n\ndisplay(deberta_large_adequate)\ndisplay(deberta_large_effective)\ndisplay(deberta_large_ineffective)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:00:52.608118Z","iopub.execute_input":"2022-08-10T02:00:52.608761Z","iopub.status.idle":"2022-08-10T02:00:52.644234Z","shell.execute_reply.started":"2022-08-10T02:00:52.608724Z","shell.execute_reply":"2022-08-10T02:00:52.643176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Submission","metadata":{}},{"cell_type":"code","source":"# Calculate the mean prediction probabilities of each folds\nsubmission = pd.read_csv('../input/feedback-prize-effectiveness/sample_submission.csv')\n# display(submission.head(20))\n\nlevel_names = ['cocolm_base', 'deberta_base', 'deberta_large']\n\nineffective_ = pd.concat(\n [cocolm_base_ineffective, deberta_base_ineffective, deberta_large_ineffective],\n keys=level_names, axis=1\n)\n\nadequate_ = pd.concat(\n [cocolm_base_adequate,deberta_base_adequate, deberta_large_adequate],\n keys=level_names, axis=1\n)\n\neffective_ = pd.concat(\n [cocolm_base_effective, deberta_base_effective, deberta_large_effective],\n keys=level_names, axis=1\n)\n\n# display(ineffective_)\n# display(adequate_)\n# display(effective_)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:01:37.884520Z","iopub.execute_input":"2022-08-10T02:01:37.884919Z","iopub.status.idle":"2022-08-10T02:01:37.907252Z","shell.execute_reply.started":"2022-08-10T02:01:37.884887Z","shell.execute_reply":"2022-08-10T02:01:37.906357Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_gradient(df, n_row=None):\n if not n_row:\n n_row = 5\n\n return df.head(n_row) \\\n .assign(all_mean=lambda x: x.mean(axis=1)) \\\n .style.background_gradient(cmap=cm, axis=1)","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:01:43.724651Z","iopub.execute_input":"2022-08-10T02:01:43.725051Z","iopub.status.idle":"2022-08-10T02:01:43.731575Z","shell.execute_reply.started":"2022-08-10T02:01:43.725018Z","shell.execute_reply":"2022-08-10T02:01:43.729976Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import seaborn as sns\n\npd.set_option('display.precision', 4)\ncm = sns.light_palette('green', as_cmap=True)\nprops_param = \"color:white; font-weight:bold; background-color:green;\"\nN_ROW = 10\n\nshow_gradient(\n ineffective_,\n N_ROW\n)\n# show_gradient(\n# adequate_,\n# N_ROW\n# )\n# show_gradient(\n# effective_,\n# N_ROW\n# )","metadata":{"execution":{"iopub.status.busy":"2022-08-10T02:01:46.435015Z","iopub.execute_input":"2022-08-10T02:01:46.435389Z","iopub.status.idle":"2022-08-10T02:01:46.569552Z","shell.execute_reply.started":"2022-08-10T02:01:46.435358Z","shell.execute_reply":"2022-08-10T02:01:46.568630Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"w_ = [0.25, 0.15, 0.6] #['cocolm_ba','deberta_ba','deberta_lar']\nd_ = [('Ineffective', ineffective_),\n ('Adequate', adequate_),\n ('Effective', effective_)]\n\nsubmission = pd.read_csv('../input/feedback-prize-effectiveness/sample_submission.csv')\ndisplay(submission.head(N_ROW))\n\nfor x in d_:\n col_name, df = x\n submission[col_name] = pd.DataFrame(\n {col: df[col].mean(axis=1) for col in level_names}\n ).mul(w_).sum(axis=1) \n\nsubmission.head(N_ROW)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:28:56.846966Z","iopub.execute_input":"2022-08-09T10:28:56.847344Z","iopub.status.idle":"2022-08-09T10:28:56.884860Z","shell.execute_reply.started":"2022-08-09T10:28:56.847312Z","shell.execute_reply":"2022-08-09T10:28:56.883831Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-09T10:29:01.133211Z","iopub.execute_input":"2022-08-09T10:29:01.134226Z","iopub.status.idle":"2022-08-09T10:29:01.142614Z","shell.execute_reply.started":"2022-08-09T10:29:01.134185Z","shell.execute_reply":"2022-08-09T10:29:01.141461Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}