{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# About this notebook\n- Deberta-v3-large starter code\n- pip wheels is [here](https://www.kaggle.com/code/yasufuminakama/pppm-pip-wheels)\n- Inference notebook is [here](https://www.kaggle.com/code/yasufuminakama/pppm-deberta-v3-large-baseline-inference)\n\nIf this notebook is helpful, feel free to upvote :)","metadata":{"id":"e460cbb5","papermill":{"duration":0.026723,"end_time":"2022-03-21T11:33:53.225849","exception":false,"start_time":"2022-03-21T11:33:53.199126","status":"completed"},"tags":[]}},{"cell_type":"code","source":"class CFG:\n    apex=True\n    print_freq=50\n    num_workers=1\n    path=\"../input/feedback-deberta-large-051/\"\n    config_path=path+'config.pth'\n    gradient_checkpoint=False\n    #model=\"anferico/bert-for-patents\"\n    #model=\"microsoft/deberta-v3-large\"\n    model = 'microsoft/deberta-v3-large'\n    scheduler='cosine' # ['linear', 'cosine']\n    batch_scheduler=True\n    num_cycles=0.5\n    num_warmup_steps=0\n    epochs=4\n    encoder_lr=5e-7\n    decoder_lr=5e-7\n    min_lr=1e-8\n    eps=1e-8\n    betas=(0.9, 0.999)\n    batch_size=4\n    fc_dropout=0.2\n    target_size=3\n    max_len=512\n    weight_decay=0.01\n    gradient_accumulation_steps=1\n    max_grad_norm=1000\n    seed=42\n    cvseed=42\n    n_fold=4\n    trn_fold=[0]\n    train=True\n    wandb=False","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:41:25.469455Z","iopub.execute_input":"2022-08-01T10:41:25.470653Z","iopub.status.idle":"2022-08-01T10:41:25.523205Z","shell.execute_reply.started":"2022-08-01T10:41:25.470018Z","shell.execute_reply":"2022-08-01T10:41:25.521851Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Library","metadata":{"id":"f2ed8ef2","papermill":{"duration":0.024963,"end_time":"2022-03-21T11:33:53.573248","exception":false,"start_time":"2022-03-21T11:33:53.548285","status":"completed"},"tags":[]}},{"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\n\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport shutil\n\nfrom torch.utils.data import DataLoader, Dataset\nimport datasets, transformers\nfrom transformers import TrainingArguments, Trainer\nfrom transformers import AutoModelForSequenceClassification, AutoTokenizer, DataCollatorWithPadding\n\nimport gc, re ,ast,sys ,copy,json,time ,math ,shutil, string, pickle, random, joblib,itertools\n\nfrom pathlib import Path\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport scipy as sp\nimport numpy as np\nimport pandas as pd\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\nfrom tqdm.auto import tqdm\nfrom sklearn.metrics import f1_score, log_loss\nfrom sklearn.model_selection import StratifiedKFold, GroupKFold, KFold\nfrom sklearn.preprocessing import LabelEncoder\n\nimport torch\nprint(f\"torch.__version__: {torch.__version__}\")\nimport torch.nn as nn\nfrom torch.nn import Parameter\nimport torch.nn.functional as F\nfrom torch.optim import Adam, SGD, AdamW\nfrom torch.utils.data import DataLoader, Dataset\n\n# os.system('pip uninstall -y transformers')\n# os.system('pip uninstall -y tokenizers')\n# os.system('python -m pip install --no-index --find-links=../input/pppm-pip-wheels transformers')\n# os.system('python -m pip install --no-index --find-links=../input/pppm-pip-wheels tokenizers')\nimport tokenizers\nimport transformers\nprint(f\"tokenizers.__version__: {tokenizers.__version__}\")\nprint(f\"transformers.__version__: {transformers.__version__}\")\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nfrom transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n%env TOKENIZERS_PARALLELISM=true\n\ngc.collect()\n\ndevice = torch.device('cuda' if torch.cuda.is_available() else 'cpu')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:41:25.530024Z","iopub.execute_input":"2022-08-01T10:41:25.533087Z","iopub.status.idle":"2022-08-01T10:41:36.367188Z","shell.execute_reply.started":"2022-08-01T10:41:25.533043Z","shell.execute_reply":"2022-08-01T10:41:36.365750Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# Utils\n# ====================================================\nINPUT_DIR = \"../input/feedback-prize-effectiveness/\"\nOUTPUT_DIR = './'\n\ndef get_logger(filename=OUTPUT_DIR+'train'):\n    from logging import getLogger, INFO, StreamHandler, FileHandler, Formatter\n    logger = getLogger(__name__)\n    logger.setLevel(INFO)\n    handler1 = StreamHandler()\n    handler1.setFormatter(Formatter(\"%(message)s\"))\n    handler2 = FileHandler(filename=f\"{filename}.log\")\n    handler2.setFormatter(Formatter(\"%(message)s\"))\n    logger.addHandler(handler1)\n    logger.addHandler(handler2)\n    return logger\n\nLOGGER = get_logger()\n\ndef get_essay(essay_id, is_train=True):\n    parent_path = INPUT_DIR + 'train' if is_train else INPUT_DIR + 'test'\n    essay_path = os.path.join(parent_path, f\"{essay_id}.txt\")\n    essay_text = open(essay_path, 'r').read()\n    return essay_text\n\ndef get_score(y_true, y_pred):\n    \n    return nn.CrossEntropyLoss()(torch.tensor(y_pred),torch.tensor(y_true))\n\ndef seed_everything(seed=CFG.seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    \nseed_everything(seed=CFG.seed)\n","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:41:36.370215Z","iopub.execute_input":"2022-08-01T10:41:36.372752Z","iopub.status.idle":"2022-08-01T10:41:36.390789Z","shell.execute_reply.started":"2022-08-01T10:41:36.372707Z","shell.execute_reply":"2022-08-01T10:41:36.389620Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Directory settings","metadata":{}},{"cell_type":"code","source":"# ====================================================\n# Directory settings & tokenizer\n# ====================================================\ntokenizer = AutoTokenizer.from_pretrained(CFG.model)\ntokenizer.save_pretrained(OUTPUT_DIR+'tokenizer/')\nCFG.tokenizer = tokenizer\n\ntrain = pd.read_csv(os.path.join(INPUT_DIR, 'train.csv'))\ntrain['essay_text']  = train['essay_id'].apply(lambda x: get_essay(x, is_train=True))\ntest = pd.read_csv(os.path.join(INPUT_DIR, 'test.csv'))\nsubmission = pd.read_csv(os.path.join(INPUT_DIR, 'sample_submission.csv'))\ntest['essay_text']  = test['essay_id'].apply(lambda x: get_essay(x, is_train=False))","metadata":{"id":"fa3b873b","papermill":{"duration":0.039998,"end_time":"2022-03-21T11:33:53.291434","exception":false,"start_time":"2022-03-21T11:33:53.251436","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-01T10:41:36.394481Z","iopub.execute_input":"2022-08-01T10:41:36.394948Z","iopub.status.idle":"2022-08-01T10:42:22.222296Z","shell.execute_reply.started":"2022-08-01T10:41:36.394907Z","shell.execute_reply":"2022-08-01T10:42:22.220814Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tokenizer","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:22.225163Z","iopub.execute_input":"2022-08-01T10:42:22.225945Z","iopub.status.idle":"2022-08-01T10:42:22.238546Z","shell.execute_reply.started":"2022-08-01T10:42:22.225903Z","shell.execute_reply":"2022-08-01T10:42:22.237196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.describe()","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:22.241031Z","iopub.execute_input":"2022-08-01T10:42:22.241461Z","iopub.status.idle":"2022-08-01T10:42:22.428741Z","shell.execute_reply.started":"2022-08-01T10:42:22.241422Z","shell.execute_reply":"2022-08-01T10:42:22.427489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#各个type的数量\nfor i in train['discourse_type'].unique():\n    print(i,':',train['discourse_type'][train['discourse_type']==i].size)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:22.430847Z","iopub.execute_input":"2022-08-01T10:42:22.431513Z","iopub.status.idle":"2022-08-01T10:42:22.484185Z","shell.execute_reply.started":"2022-08-01T10:42:22.431468Z","shell.execute_reply":"2022-08-01T10:42:22.482918Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nfig = plt.figure(figsize = (10.8,7.2))\nfig = plt.hist(train['discourse_type'])","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:22.486827Z","iopub.execute_input":"2022-08-01T10:42:22.487708Z","iopub.status.idle":"2022-08-01T10:42:22.760367Z","shell.execute_reply.started":"2022-08-01T10:42:22.487666Z","shell.execute_reply":"2022-08-01T10:42:22.758920Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data Loading","metadata":{"id":"cb3d8e1e","papermill":{"duration":0.028943,"end_time":"2022-03-21T11:34:21.290864","exception":false,"start_time":"2022-03-21T11:34:21.261921","status":"completed"},"tags":[]}},{"cell_type":"code","source":"from text_unidecode import unidecode\nfrom typing import Dict, List, Tuple\nimport codecs\n\ndef replace_encoding_with_utf8(error: UnicodeError) -> Tuple[bytes, int]:\n    return error.object[error.start : error.end].encode(\"utf-8\"), error.end\n\n\ndef replace_decoding_with_cp1252(error: UnicodeError) -> Tuple[str, int]:\n    return error.object[error.start : error.end].decode(\"cp1252\"), error.end\n\n# Register the encoding and decoding error handlers for `utf-8` and `cp1252`.\ncodecs.register_error(\"replace_encoding_with_utf8\", replace_encoding_with_utf8)\ncodecs.register_error(\"replace_decoding_with_cp1252\", replace_decoding_with_cp1252)\n\ndef resolve_encodings_and_normalize(text: str) -> str:\n    \"\"\"Resolve the encoding problems and normalize the abnormal characters.\"\"\"\n    text = (\n        text.encode(\"raw_unicode_escape\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n        .encode(\"cp1252\", errors=\"replace_encoding_with_utf8\")\n        .decode(\"utf-8\", errors=\"replace_decoding_with_cp1252\")\n    )\n    text = unidecode(text)\n    return text","metadata":{"executionInfo":{"elapsed":2627,"status":"ok","timestamp":1644920084001,"user":{"displayName":"Yasufumi Nakama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"17486303986134302670"},"user_tz":-540},"id":"bef012d3","outputId":"d4d60dbc-510c-4f34-8d64-dd1d88c4808c","papermill":{"duration":0.860612,"end_time":"2022-03-21T11:34:22.180396","exception":false,"start_time":"2022-03-21T11:34:21.319784","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-01T10:42:22.761947Z","iopub.execute_input":"2022-08-01T10:42:22.763301Z","iopub.status.idle":"2022-08-01T10:42:22.785215Z","shell.execute_reply.started":"2022-08-01T10:42:22.763257Z","shell.execute_reply":"2022-08-01T10:42:22.784068Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"SEP = tokenizer.sep_token\n\ntrain['text'] = train['discourse_type'] + ' ' + train['discourse_text'] + SEP + train['essay_text']\ntrain['label'] = train['discourse_effectiveness']\n\ntest['text'] = test['discourse_type'] + ' ' + test['discourse_text'] + SEP + test['essay_text']\ntest['label'] = np.nan\n\ndisplay(train.head())","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:22.790368Z","iopub.execute_input":"2022-08-01T10:42:22.790677Z","iopub.status.idle":"2022-08-01T10:42:22.926090Z","shell.execute_reply.started":"2022-08-01T10:42:22.790648Z","shell.execute_reply":"2022-08-01T10:42:22.924825Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# CV split","metadata":{"id":"9e05b6c4","papermill":{"duration":0.031837,"end_time":"2022-03-21T11:34:22.593468","exception":false,"start_time":"2022-03-21T11:34:22.561631","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# CV split\n# ====================================================\n#train['score_map'] = train['score'].map({0.00: 0, 0.25: 1, 0.50: 2, 0.75: 3, 1.00: 4})\nFold = StratifiedKFold(n_splits=CFG.n_fold, shuffle=True, random_state=CFG.cvseed)\nfor n, (train_index, val_index) in enumerate(Fold.split(train, train['discourse_effectiveness'])):\n    train.loc[val_index, 'fold'] = int(n)\ntrain['fold'] = train['fold'].astype(int)\ndisplay(train.groupby('fold').size())","metadata":{"executionInfo":{"elapsed":12,"status":"ok","timestamp":1644920084528,"user":{"displayName":"Yasufumi Nakama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"17486303986134302670"},"user_tz":-540},"id":"3ba287c4","outputId":"307dc0e2-17d6-4bfe-9e95-fdd9b6303974","papermill":{"duration":0.052109,"end_time":"2022-03-21T11:34:22.677648","exception":false,"start_time":"2022-03-21T11:34:22.625539","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-01T10:42:22.928940Z","iopub.execute_input":"2022-08-01T10:42:22.930328Z","iopub.status.idle":"2022-08-01T10:42:23.010292Z","shell.execute_reply.started":"2022-08-01T10:42:22.930279Z","shell.execute_reply":"2022-08-01T10:42:23.008805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.groupby('fold')['discourse_effectiveness'].value_counts()#分布均匀","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:23.011931Z","iopub.execute_input":"2022-08-01T10:42:23.012471Z","iopub.status.idle":"2022-08-01T10:42:23.032715Z","shell.execute_reply.started":"2022-08-01T10:42:23.012428Z","shell.execute_reply":"2022-08-01T10:42:23.031607Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"encoder = LabelEncoder()\ntrain['discourse_effectiveness'] = encoder.fit_transform(train['discourse_effectiveness'])","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:23.034539Z","iopub.execute_input":"2022-08-01T10:42:23.034950Z","iopub.status.idle":"2022-08-01T10:42:23.054620Z","shell.execute_reply.started":"2022-08-01T10:42:23.034912Z","shell.execute_reply":"2022-08-01T10:42:23.053237Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Dataset","metadata":{"id":"14da40cf","papermill":{"duration":0.034425,"end_time":"2022-03-21T11:34:28.504726","exception":false,"start_time":"2022-03-21T11:34:28.470301","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Dataset\n# ====================================================\nclass TrainDataset(Dataset):\n    def __init__(self, cfg, df):\n        self.cfg = cfg\n        self.text = df['text'].values\n        self.labels = df['discourse_effectiveness'].values\n\n    def __len__(self):\n        return len(self.text)\n\n    def __getitem__(self, item):\n        inputs = self.cfg.tokenizer.encode_plus(\n                        self.text[item],\n                        truncation=True,\n                        add_special_tokens=True,\n                        max_length=self.cfg.max_len,\n                        padding=\"max_length\"\n                    )\n        \n        for k, v in inputs.items():\n            inputs[k] = torch.tensor(v, dtype=torch.long)\n        \n        if 'token_type_ids' in inputs:\n            inputs['token_type_ids'] = inputs['token_type_ids']\n            \n        labels = self.labels[item]\n            \n        \n        return inputs, labels","metadata":{"id":"9f791a19","papermill":{"duration":0.053757,"end_time":"2022-03-21T11:34:55.302054","exception":false,"start_time":"2022-03-21T11:34:55.248297","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-01T10:42:23.056712Z","iopub.execute_input":"2022-08-01T10:42:23.057203Z","iopub.status.idle":"2022-08-01T10:42:23.070703Z","shell.execute_reply.started":"2022-08-01T10:42:23.057163Z","shell.execute_reply":"2022-08-01T10:42:23.069441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class MeanPooling(nn.Module):\n    def __init__(self):\n        super(MeanPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()\n        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n        sum_mask = input_mask_expanded.sum(1)\n        sum_mask = torch.clamp(sum_mask, min=1e-9)\n        mean_embeddings = sum_embeddings / sum_mask\n        return mean_embeddings\n\n    \nclass MeanMaxPooling(nn.Module):\n    def __init__(self):\n        super(MeanMaxPooling, self).__init__()\n        \n    def forward(self, last_hidden_state, attention_mask):\n        mean_pooling_embeddings = torch.mean(last_hidden_state, 1)\n        _, max_pooling_embeddings = torch.max(last_hidden_state, 1)\n        mean_max_embeddings = torch.cat((mean_pooling_embeddings, max_pooling_embeddings), 1)\n        return mean_max_embeddings\n\n    \nclass LSTMPooling(nn.Module):\n    def __init__(self, num_layers, hidden_size, hiddendim_lstm):\n        super(LSTMPooling, self).__init__()\n        self.num_hidden_layers = num_layers\n        self.hidden_size = hidden_size\n        self.hiddendim_lstm = hiddendim_lstm\n        self.lstm = nn.LSTM(self.hidden_size, self.hiddendim_lstm, batch_first=True)\n        self.dropout = nn.Dropout(0.1)\n    \n    def forward(self, all_hidden_states):\n        ## forward\n        hidden_states = torch.stack([all_hidden_states[layer_i][:, 0].squeeze()\n                                     for layer_i in range(1, self.num_hidden_layers+1)], dim=-1)\n        hidden_states = hidden_states.view(-1, self.num_hidden_layers, self.hidden_size)\n        out, _ = self.lstm(hidden_states, None)\n        out = self.dropout(out[:, -1, :])\n        return out\n    \nclass WeightedLayerPooling(nn.Module):\n    def __init__(self, num_hidden_layers, layer_start: int = 4, layer_weights = None):\n        super(WeightedLayerPooling, self).__init__()\n        self.layer_start = layer_start\n        self.num_hidden_layers = num_hidden_layers\n        self.layer_weights = layer_weights if layer_weights is not None \\\n            else nn.Parameter(\n                torch.tensor([1] * (num_hidden_layers+1 - layer_start), dtype=torch.float)\n            )\n\n    def forward(self, all_hidden_states):\n        all_layer_embedding = all_hidden_states[self.layer_start:, :, :, :]\n        weight_factor = self.layer_weights.unsqueeze(-1).unsqueeze(-1).unsqueeze(-1).expand(all_layer_embedding.size())\n        weighted_average = (weight_factor*all_layer_embedding).sum(dim=0) / self.layer_weights.sum()\n        return weighted_average","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:23.073085Z","iopub.execute_input":"2022-08-01T10:42:23.073538Z","iopub.status.idle":"2022-08-01T10:42:23.095677Z","shell.execute_reply.started":"2022-08-01T10:42:23.073500Z","shell.execute_reply":"2022-08-01T10:42:23.094393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\"\"\"\ntrain_dataset = TrainDataset(CFG, train)\ninputs, label = train_dataset[0]\nprint(inputs)\nprint(label)\n\"\"\"","metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1644920122808,"user":{"displayName":"Yasufumi Nakama","photoUrl":"https://lh3.googleusercontent.com/a/default-user=s64","userId":"17486303986134302670"},"user_tz":-540},"id":"a200bd5b","outputId":"a30fde2b-86f9-4ebd-ae81-33f467b69836","papermill":{"duration":0.043429,"end_time":"2022-03-21T11:34:55.463328","exception":false,"start_time":"2022-03-21T11:34:55.419899","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-01T10:42:23.097836Z","iopub.execute_input":"2022-08-01T10:42:23.098696Z","iopub.status.idle":"2022-08-01T10:42:23.117268Z","shell.execute_reply.started":"2022-08-01T10:42:23.098654Z","shell.execute_reply":"2022-08-01T10:42:23.115961Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Model","metadata":{"id":"e04d6363","papermill":{"duration":0.036359,"end_time":"2022-03-21T11:34:55.535904","exception":false,"start_time":"2022-03-21T11:34:55.499545","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Model\n# ====================================================\n\nfrom torch.cuda.amp import autocast\nclass CustomModel(nn.Module):\n    def __init__(self, cfg, config_path=None, pretrained=False):\n        super().__init__()\n        self.cfg = cfg\n        if config_path is None:\n            self.config = AutoConfig.from_pretrained(cfg.model, output_hidden_states=True)\n        else:\n            self.config = torch.load(config_path)\n        \n        if pretrained:\n            self.model = AutoModel.from_pretrained(cfg.model, config=self.config)\n        else:\n            self.model = AutoModel.from_config(self.config)\n        \n        # gradient checkpointing\n        if self.cfg.gradient_checkpoint:\n            self.model.gradient_checkpointing_enable()\n            print(f\"Gradient Checkpointing: {self.model.is_gradient_checkpointing}\")\n            \n        \n        # self.pooler = MeanPooling()\n        \n        self.bilstm = nn.LSTM(self.config.hidden_size, (self.config.hidden_size) // 2, num_layers=2, \n                              dropout=self.config.hidden_dropout_prob, batch_first=True,\n                              bidirectional=True)\n        \n        self.dropout = nn.Dropout(0.2)\n        self.dropout1 = nn.Dropout(0.1)\n        self.dropout2 = nn.Dropout(0.2)\n        self.dropout3 = nn.Dropout(0.3)\n        self.dropout4 = nn.Dropout(0.4)\n        self.dropout5 = nn.Dropout(0.5)\n        \n        self.output = nn.Sequential(\n            nn.Linear(self.config.hidden_size, self.cfg.target_size)\n            # nn.Linear(256, self.cfg.target_size)\n        )\n        \n\n        \n    def _init_weights(self, module):\n        if isinstance(module, nn.Linear):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.bias is not None:\n                module.bias.data.zero_()\n        elif isinstance(module, nn.Embedding):\n            module.weight.data.normal_(mean=0.0, std=self.config.initializer_range)\n            if module.padding_idx is not None:\n                module.weight.data[module.padding_idx].zero_()\n        elif isinstance(module, nn.LayerNorm):\n            module.bias.data.zero_()\n            module.weight.data.fill_(1.0)\n\n    def forward(self, inputs):\n        transformer_out = self.model(**inputs)\n        \n        # LSTM/GRU header\n#         all_hidden_states = torch.stack(transformer_out[1])\n#         sequence_output = self.pooler(all_hidden_states)\n        \n        # simple CLS\n        sequence_output = transformer_out[0][:, 0, :]\n\n        \n        # Main task\n        logits1 = self.output(self.dropout1(sequence_output))\n        logits2 = self.output(self.dropout2(sequence_output))\n        logits3 = self.output(self.dropout3(sequence_output))\n        logits4 = self.output(self.dropout4(sequence_output))\n        logits5 = self.output(self.dropout5(sequence_output))\n        logits = (logits1 + logits2 + logits3 + logits4 + logits5) / 5\n        \n        return logits\n    \ndef inplace_relu(m):\n    classname = m.__class__.__name__\n    if classname.find('ReLU') != -1:\n        m.inplace=True\n\n\n","metadata":{"id":"4c5bab44","papermill":{"duration":0.05059,"end_time":"2022-03-21T11:34:55.622912","exception":false,"start_time":"2022-03-21T11:34:55.572322","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-01T10:42:23.119506Z","iopub.execute_input":"2022-08-01T10:42:23.120182Z","iopub.status.idle":"2022-08-01T10:42:23.143739Z","shell.execute_reply.started":"2022-08-01T10:42:23.119963Z","shell.execute_reply":"2022-08-01T10:42:23.142228Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Helpler functions","metadata":{"id":"deee9675","papermill":{"duration":0.03649,"end_time":"2022-03-21T11:34:55.940757","exception":false,"start_time":"2022-03-21T11:34:55.904267","status":"completed"},"tags":[]}},{"cell_type":"code","source":"# ====================================================\n# Helper functions\n# ====================================================\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef asMinutes(s):\n    m = math.floor(s / 60)\n    s -= m * 60\n    return '%dm %ds' % (m, s)\n\n\ndef timeSince(since, percent):\n    now = time.time()\n    s = now - since\n    es = s / (percent)\n    rs = es - s\n    return '%s (remain %s)' % (asMinutes(s), asMinutes(rs))\n\n\ndef train_fn(fold, train_loader, model, criterion, optimizer, epoch, scheduler, device):\n    model.train()\n    scaler = torch.cuda.amp.GradScaler(enabled=CFG.apex)\n    losses = AverageMeter()\n    start = end = time.time()\n    global_step = 0\n    for step, (inputs, labels) in enumerate(train_loader):\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        with torch.cuda.amp.autocast(enabled=CFG.apex):\n            y_preds = model(inputs)\n            loss = criterion(y_preds, labels)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        losses.update(loss.item(), batch_size)\n        scaler.scale(loss).backward()\n        grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), CFG.max_grad_norm)\n        if (step + 1) % CFG.gradient_accumulation_steps == 0:\n            scaler.step(optimizer)\n            scaler.update()\n            optimizer.zero_grad()\n            global_step += 1\n            if CFG.batch_scheduler:\n                scheduler.step()\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(train_loader)-1):\n            print('Epoch: [{0}][{1}/{2}] '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  'Grad: {grad_norm:.4f}  '\n                  'LR: {lr:.8f}  '\n                  .format(epoch+1, step, len(train_loader), \n                          remain=timeSince(start, float(step+1)/len(train_loader)),\n                          loss=losses,\n                          grad_norm=grad_norm,\n                          lr=scheduler.get_lr()[0]))\n        if CFG.wandb:\n            wandb.log({f\"[fold{fold}] loss\": losses.val,\n                       f\"[fold{fold}] lr\": scheduler.get_lr()[0]})\n    return losses.avg\n\n\ndef valid_fn(valid_loader, model, criterion, device):\n    losses = AverageMeter()\n    model.eval()\n    #preds = np.array([])\n    preds = []\n    start = end = time.time()\n    for step, (inputs, labels) in enumerate(valid_loader):\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n        labels = labels.to(device)\n        batch_size = labels.size(0)\n        with torch.no_grad():\n            y_preds = model(inputs)\n        loss = criterion(y_preds, labels)\n        if CFG.gradient_accumulation_steps > 1:\n            loss = loss / CFG.gradient_accumulation_steps\n        losses.update(loss.item(), batch_size)\n        preds.append(y_preds.to('cpu').numpy())\n        \n#         if step > 0:\n#             preds = np.row_stack((preds, y_preds.sigmoid().to('cpu').numpy()))\n#         else:\n#             preds = y_preds.sigmoid().to('cpu').numpy()\n\n        end = time.time()\n        if step % CFG.print_freq == 0 or step == (len(valid_loader)-1):\n            print('EVAL: [{0}/{1}] '\n                  'Elapsed {remain:s} '\n                  'Loss: {loss.val:.4f}({loss.avg:.4f}) '\n                  .format(step, len(valid_loader),\n                          loss=losses,\n                          remain=timeSince(start, float(step+1)/len(valid_loader))))\n    predictions = np.concatenate(preds)\n    #print(preds.shape, predictions.shape)\n#     predictions = np.concatenate(predictions)\n    return losses.avg, predictions\n\n\ndef inference_fn(test_loader, model, device):\n    preds = []\n    model.eval()\n    model.to(device)\n    tk0 = tqdm(test_loader, total=len(test_loader))\n    for inputs in tk0:\n        for k, v in inputs.items():\n            inputs[k] = v.to(device)\n        with torch.no_grad():\n            y_preds = model(inputs)\n        preds.append(y_preds.to('cpu').numpy())\n    predictions = np.concatenate(preds)\n    return predictions","metadata":{"id":"c8263b0c","papermill":{"duration":0.112662,"end_time":"2022-03-21T11:34:56.089768","exception":false,"start_time":"2022-03-21T11:34:55.977106","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-01T10:42:23.146087Z","iopub.execute_input":"2022-08-01T10:42:23.146763Z","iopub.status.idle":"2022-08-01T10:42:23.181208Z","shell.execute_reply.started":"2022-08-01T10:42:23.146724Z","shell.execute_reply":"2022-08-01T10:42:23.179830Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test code\n# fold = 0\n# folds = train\n\n# train_folds = folds[folds['fold'] != fold].reset_index(drop=True)\n# valid_folds = folds[folds['fold'] == fold].reset_index(drop=True)\n# valid_labels = valid_folds['discourse_effectiveness'].values\n\n\n# train_dataset = TrainDataset(CFG, train_folds)\n# valid_dataset = TrainDataset(CFG, valid_folds)\n\n# train_loader = DataLoader(train_dataset,\n#                           batch_size=CFG.batch_size,\n#                           shuffle=True,\n#                           num_workers=CFG.num_workers, pin_memory=True, drop_last=True)\n# valid_loader = DataLoader(valid_dataset,\n#                           batch_size=CFG.batch_size,\n#                           shuffle=False,\n#                           num_workers=CFG.num_workers, pin_memory=True, drop_last=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:23.184404Z","iopub.execute_input":"2022-08-01T10:42:23.185213Z","iopub.status.idle":"2022-08-01T10:42:23.200402Z","shell.execute_reply.started":"2022-08-01T10:42:23.185171Z","shell.execute_reply":"2022-08-01T10:42:23.199114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model = CustomModel(CFG, config_path=None, pretrained=True)\n# model.to(device)\n\n# dataiter = iter(train_loader)\n# x,y = next(dataiter)\n# y.to(device)\n# for k, v in x.items():\n#     x[k] = v.to(device)\n    \n# y = y.to(device)\n# y_hat = model(x)\n# loss = nn.CrossEntropyLoss().to(device)\n# loss(y_hat, y)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:23.203108Z","iopub.execute_input":"2022-08-01T10:42:23.204207Z","iopub.status.idle":"2022-08-01T10:42:23.214798Z","shell.execute_reply.started":"2022-08-01T10:42:23.204164Z","shell.execute_reply":"2022-08-01T10:42:23.213130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# a = torch.randint(0,3,(1,50))[0]\n# b = nn.Softmax()(torch.rand([50,3]))\n# nn.CrossEntropyLoss()(b, a)","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:42:23.218514Z","iopub.execute_input":"2022-08-01T10:42:23.219906Z","iopub.status.idle":"2022-08-01T10:42:23.227068Z","shell.execute_reply.started":"2022-08-01T10:42:23.219832Z","shell.execute_reply":"2022-08-01T10:42:23.225737Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# ====================================================\n# train loop\n# ====================================================\ndef train_loop(folds, fold):\n    \n    LOGGER.info(f\"========== fold: {fold} training ==========\")\n\n    # ====================================================\n    # loader\n    # ====================================================\n    train_folds = folds[folds['fold'] != fold].reset_index(drop=True)\n    valid_folds = folds[folds['fold'] == fold].reset_index(drop=True)\n    valid_labels = valid_folds['discourse_effectiveness'].values\n#     for i in valid_folds['discourse_effectiveness'].values:\n#         if i == 0:\n#             valid_labels.append([1,0,0])\n#         elif i == 1:\n#             valid_labels.append([0,1,0])\n#         elif i == 2:\n#             valid_labels.append([0,0,1])\n    \n    train_dataset = TrainDataset(CFG, train_folds)\n    valid_dataset = TrainDataset(CFG, valid_folds)\n\n    train_loader = DataLoader(train_dataset,\n                              batch_size=CFG.batch_size,\n                              shuffle=True,\n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=True)\n    valid_loader = DataLoader(valid_dataset,\n                              batch_size=CFG.batch_size,\n                              shuffle=False,\n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n\n    # ====================================================\n    # model & optimizer\n    # ====================================================\n    model = CustomModel(CFG, config_path=None, pretrained=True)\n    #model.apply(inplace_relu)\n    torch.save(model.config, OUTPUT_DIR+'config.pth')\n    model.to(device)\n    \n    def get_optimizer_params(model, encoder_lr, decoder_lr, weight_decay=0.0):\n        param_optimizer = list(model.named_parameters())\n        no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n        optimizer_parameters = [\n            {'params': [p for n, p in model.model.named_parameters() if not any(nd in n for nd in no_decay)],\n             'lr': encoder_lr, 'weight_decay': weight_decay},\n            {'params': [p for n, p in model.model.named_parameters() if any(nd in n for nd in no_decay)],\n             'lr': encoder_lr, 'weight_decay': 0.0},\n            {'params': [p for n, p in model.named_parameters() if \"model\" not in n],\n             'lr': decoder_lr, 'weight_decay': 0.0}\n        ]\n        return optimizer_parameters\n\n    optimizer_parameters = get_optimizer_params(model,\n                                                encoder_lr=CFG.encoder_lr, \n                                                decoder_lr=CFG.decoder_lr,\n                                                weight_decay=CFG.weight_decay)\n    optimizer = AdamW(optimizer_parameters, lr=CFG.encoder_lr, eps=CFG.eps, betas=CFG.betas)\n    \n    # ====================================================\n    # scheduler\n    # ====================================================\n    def get_scheduler(cfg, optimizer, num_train_steps):\n        if cfg.scheduler == 'linear':\n            scheduler = get_linear_schedule_with_warmup(\n                optimizer, num_warmup_steps=cfg.num_warmup_steps, num_training_steps=num_train_steps\n            )\n        elif cfg.scheduler == 'cosine':\n            scheduler = get_cosine_schedule_with_warmup(\n                optimizer, num_warmup_steps=cfg.num_warmup_steps, num_training_steps=num_train_steps, num_cycles=cfg.num_cycles\n            )\n        return scheduler\n    \n    num_train_steps = int(len(train_folds) / CFG.batch_size * CFG.epochs)\n    scheduler = get_scheduler(CFG, optimizer, num_train_steps)\n\n    # ====================================================\n    # loop\n    # ====================================================\n    #criterion = nn.BCEWithLogitsLoss(reduction=\"mean\")\n    criterion = nn.CrossEntropyLoss()\n    \n    best_score = 100.\n\n    for epoch in range(CFG.epochs):\n\n        start_time = time.time()\n\n        # train\n        avg_loss = train_fn(fold, train_loader, model, criterion, optimizer, epoch, scheduler, device)\n\n        # eval\n        avg_val_loss, predictions = valid_fn(valid_loader, model, criterion, device)\n        \n        # scoring\n        #print(valid_labels.shape, predictions.shape)\n        score = get_score(valid_labels, predictions)\n\n        elapsed = time.time() - start_time\n\n        LOGGER.info(f'Epoch {epoch+1} - avg_train_loss: {avg_loss:.4f}  avg_val_loss: {avg_val_loss:.4f}  time: {elapsed:.0f}s')\n        LOGGER.info(f'Epoch {epoch+1} - Score: {score:.4f}')\n        if CFG.wandb:\n            wandb.log({f\"[fold{fold}] epoch\": epoch+1, \n                       f\"[fold{fold}] avg_train_loss\": avg_loss, \n                       f\"[fold{fold}] avg_val_loss\": avg_val_loss,\n                       f\"[fold{fold}] score\": score})\n        \n        if best_score > score:\n            best_score = score\n            LOGGER.info(f'Epoch {epoch+1} - Save Best Score: {best_score:.4f} Model')\n            torch.save({'model': model.state_dict(),\n                        'predictions': predictions},\n                        OUTPUT_DIR+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\")\n\n    predictions = torch.load(OUTPUT_DIR+f\"{CFG.model.replace('/', '-')}_fold{fold}_best.pth\", \n                             map_location=torch.device('cpu'))['predictions']\n    valid_folds[['pred_0','pred_1','pred_2']] = predictions\n\n    torch.cuda.empty_cache()\n    gc.collect()\n    \n    return valid_folds","metadata":{"id":"bed940e1","papermill":{"duration":0.300071,"end_time":"2022-03-21T11:34:56.447127","exception":false,"start_time":"2022-03-21T11:34:56.147056","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-01T10:42:23.230866Z","iopub.execute_input":"2022-08-01T10:42:23.231328Z","iopub.status.idle":"2022-08-01T10:42:23.261915Z","shell.execute_reply.started":"2022-08-01T10:42:23.231282Z","shell.execute_reply":"2022-08-01T10:42:23.260506Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"if __name__ == '__main__':\n    \n    def get_result(oof_df):\n        labels = oof_df['discourse_effectiveness'].values\n        preds = oof_df[['pred_0','pred_1','pred_2']].values\n        score = get_score(labels, preds)\n        LOGGER.info(f'Score: {score:<.4f}')\n    \n    oof_df = pd.DataFrame()\n    for fold in range(CFG.n_fold):\n        if fold in CFG.trn_fold:\n            _oof_df = train_loop(train, fold)\n            oof_df = pd.concat([oof_df, _oof_df])\n            LOGGER.info(f\"========== fold: {fold} result ==========\")\n            get_result(_oof_df)\n    oof_df = oof_df.reset_index(drop=True)\n    LOGGER.info(f\"========== CV ==========\")\n    get_result(oof_df)\n    oof_df.to_pickle(OUTPUT_DIR+'oof_df.pkl')\n        \n    if CFG.wandb:\n        wandb.finish()","metadata":{"id":"6cc76b1e","papermill":{"duration":6464.068626,"end_time":"2022-03-21T13:22:40.573692","exception":false,"start_time":"2022-03-21T11:34:56.505066","status":"completed"},"tags":[],"execution":{"iopub.status.busy":"2022-08-01T10:42:23.264047Z","iopub.execute_input":"2022-08-01T10:42:23.264928Z","iopub.status.idle":"2022-08-01T10:43:53.772743Z","shell.execute_reply.started":"2022-08-01T10:42:23.264872Z","shell.execute_reply":"2022-08-01T10:43:53.768938Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#test\n# ====================================================\n# loader\n# ====================================================\nif 0:\n    fold = 0\n    folds = train\n\n    train_folds = folds[folds['fold'] != fold].reset_index(drop=True).iloc[:20]\n    valid_folds = folds[folds['fold'] == fold].reset_index(drop=True).iloc[:20]\n    valid_labels = valid_folds['discourse_effectiveness'].values\n    #     for i in valid_folds['discourse_effectiveness'].values:\n    #         if i == 0:\n    #             valid_labels.append([1,0,0])\n    #         elif i == 1:\n    #             valid_labels.append([0,1,0])\n    #         elif i == 2:\n    #             valid_labels.append([0,0,1])\n\n    train_dataset = TrainDataset(CFG, train_folds)\n    valid_dataset = TrainDataset(CFG, valid_folds)\n\n    train_loader = DataLoader(train_dataset,\n                              batch_size=CFG.batch_size,\n                              shuffle=True,\n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n    valid_loader = DataLoader(valid_dataset,\n                              batch_size=CFG.batch_size,\n                              shuffle=False,\n                              num_workers=CFG.num_workers, pin_memory=True, drop_last=False)\n\n    criterion = nn.BCEWithLogitsLoss(reduction=\"mean\")\n\n    model = CustomModel(CFG, config_path=None, pretrained=True).to(device)\n    torch.save(model.config, OUTPUT_DIR+'config.pth')","metadata":{"execution":{"iopub.status.busy":"2022-08-01T10:43:53.777223Z","iopub.status.idle":"2022-08-01T10:43:53.779632Z","shell.execute_reply.started":"2022-08-01T10:43:53.779273Z","shell.execute_reply":"2022-08-01T10:43:53.779306Z"},"trusted":true},"execution_count":null,"outputs":[]}]}