{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Config","metadata":{}},{"cell_type":"code","source":"!ls /kaggle/input","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:43:33.583501Z","iopub.execute_input":"2022-02-06T19:43:33.584305Z","iopub.status.idle":"2022-02-06T19:43:34.330124Z","shell.execute_reply.started":"2022-02-06T19:43:33.584196Z","shell.execute_reply":"2022-02-06T19:43:34.329220Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip /kaggle/input/jigsaw-toxic-comment-classification-challenge/train.csv.zip\n!unzip /kaggle/input/jigsaw-toxic-comment-classification-challenge/test.csv.zip\n!unzip /kaggle/input/jigsaw-toxic-comment-classification-challenge/test_labels.csv.zip","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:43:34.332191Z","iopub.execute_input":"2022-02-06T19:43:34.332533Z","iopub.status.idle":"2022-02-06T19:43:38.218575Z","shell.execute_reply.started":"2022-02-06T19:43:34.332498Z","shell.execute_reply":"2022-02-06T19:43:38.217755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MODEL_PATH = '/kaggle/input/jigsaw-get-zsc-models/DeBERTa-v3-base-mnli-fever-anli'\nDATA_PATH = '/kaggle/input/jigsaw-filter-pseudolabel/validation_data_std_2.0.csv'\nVAL_PCT = 0.1\nSEED = 42\n\nBATCH_SIZE = 16\nLR = 1e-4 # from auto LR finder suggestion\nWEIGHT_DECAY = 0.01\nWARMUP_RATIO = 0.06\nEPOCH = 2\nSTEPS = 152205 // BATCH_SIZE + 1 # first INT value is total training data","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:43:38.220363Z","iopub.execute_input":"2022-02-06T19:43:38.220651Z","iopub.status.idle":"2022-02-06T19:43:38.227095Z","shell.execute_reply.started":"2022-02-06T19:43:38.220615Z","shell.execute_reply":"2022-02-06T19:43:38.225701Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import torch\nimport pandas as pd\nimport pytorch_lightning as pl\nimport transformers\nimport sklearn\nimport gc\n\nprint(torch.__version__)\nprint(pd.__version__)\nprint(transformers.__version__)\nprint(pl.__version__)\nprint(sklearn.__version__)\n\npl.seed_everything(SEED, workers=True)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-02-06T19:43:38.229275Z","iopub.execute_input":"2022-02-06T19:43:38.229705Z","iopub.status.idle":"2022-02-06T19:43:47.609454Z","shell.execute_reply.started":"2022-02-06T19:43:38.229672Z","shell.execute_reply":"2022-02-06T19:43:47.608729Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Model","metadata":{}},{"cell_type":"code","source":"from transformers import AutoModel, AutoTokenizer, AutoConfig\n\nm = AutoModel.from_pretrained(MODEL_PATH)\ntokenizer = AutoTokenizer.from_pretrained(MODEL_PATH)\nconfig = AutoConfig.from_pretrained(MODEL_PATH)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:43:47.611013Z","iopub.execute_input":"2022-02-06T19:43:47.612005Z","iopub.status.idle":"2022-02-06T19:43:55.930672Z","shell.execute_reply.started":"2022-02-06T19:43:47.611964Z","shell.execute_reply":"2022-02-06T19:43:55.929895Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from transformers.models.deberta_v2.modeling_deberta_v2 import StableDropout, ContextPooler\nfrom torch.optim.lr_scheduler import OneCycleLR\n\nclass JigsawModel(pl.LightningModule):\n    def __init__(self, model):\n        super().__init__()\n        self.deberta = model\n        self.dense = torch.nn.Sequential(\n            torch.nn.Linear(768, 768),\n            StableDropout(drop_prob=0.1), # original 0.0 ??\n            torch.nn.Linear(768, 1)\n        )\n        self.loss = torch.nn.BCEWithLogitsLoss()\n    \n    def forward(self, ids, mask, token_type_ids):\n        out = self.deberta(ids, attention_mask = mask, token_type_ids = token_type_ids)\n        out = out.last_hidden_state[:, 0]\n        out = self.dense(out)\n        out = torch.reshape(out, (-1, ))\n\n        return out\n\n    def configure_optimizers(self):\n        optimizer = transformers.AdamW(self.parameters(), lr=LR, weight_decay=WEIGHT_DECAY)\n#         scheduler = transformers.get_linear_schedule_with_warmup(\n#             optimizer,\n#             int(EPOCH * STEPS * WARMUP_RATIO),\n#             int(EPOCH * STEPS * (1 - WARMUP_RATIO))\n#         )\n        scheduler = torch.optim.lr_scheduler.OneCycleLR(\n            optimizer, max_lr=LR, steps_per_epoch=STEPS, epochs=EPOCH,\n            div_factor=10, final_div_factor=40\n        )\n    \n        return [optimizer], [scheduler]\n    \n    def training_step(self, batch, batch_idx):\n        ids = batch['ids']\n        mask = batch['mask']\n        token_type_ids = batch['token_type_ids']\n        label = batch['label']\n\n        out = self(ids, mask, token_type_ids)\n        loss = self.loss(out, label)\n        \n        return loss\n\n    def validation_step(self, batch, batch_idx):\n        ids = batch['ids']\n        mask = batch['mask']\n        token_type_ids = batch['token_type_ids']\n        label = batch['label']\n\n        out = self(ids, mask, token_type_ids)\n        loss = self.loss(out, label)\n        self.log(\"val_loss\", loss, prog_bar=True)\n        \n        return loss\n\n    def training_epoch_end(self, outputs):\n        # manually print loss on each epoch\n        losses = [d['loss'] for d in outputs]\n        avg_loss = torch.stack(losses).mean()\n        print(f'Epoch #{self.current_epoch} | loss: {avg_loss}')\n\n    def validation_epoch_end(self, outputs):\n        # manually print loss on each epoch\n        avg_loss = torch.stack(outputs).mean()\n        print(f'Epoch #{self.current_epoch} | val_loss: {avg_loss}')\n        self.log(\"val_loss\", avg_loss)\n\n        \n    def predict_step(self, batch, batch_idx):\n        ids = batch['ids']\n        mask = batch['mask']\n        token_type_ids = batch['token_type_ids']\n\n        out = self(ids, mask, token_type_ids)\n        out = torch.sigmoid(out)\n\n        return out\n\n    \nmodel = JigsawModel(m)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:43:55.932152Z","iopub.execute_input":"2022-02-06T19:43:55.932416Z","iopub.status.idle":"2022-02-06T19:43:55.966484Z","shell.execute_reply.started":"2022-02-06T19:43:55.932378Z","shell.execute_reply":"2022-02-06T19:43:55.965743Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Data","metadata":{}},{"cell_type":"code","source":"# load\ndf_raw = pd.read_csv(DATA_PATH)\n# remove duplicate\ndf_raw = df_raw[df_raw.duplicated(subset=['less_toxic']) == False]\ndf_raw = df_raw[df_raw.duplicated(subset=['more_toxic']) == False]\n# create fresh dataframe\ndf = pd.DataFrame(columns=['text', 'label'])\n# select text without score \"-\"\ndf = pd.concat(\n    [\n        df,\n        (\n            df_raw.query(\"less_toxic_score != '-'\")\n                  .loc[:, ['less_toxic', 'less_toxic_score']]\n                  .rename(columns={'less_toxic': 'text', 'less_toxic_score': 'label'})\n        ),\n        (\n            df_raw.query(\"more_toxic_score != '-'\")\n                  .loc[:, ['more_toxic', 'more_toxic_score']]\n                  .rename(columns={'more_toxic': 'text', 'more_toxic_score': 'label'})\n        ),        \n    ],\n    axis=0\n)\n# set label data type\ndf['label'] = df['label'].astype('float64')\n# show\ndf","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:43:55.967751Z","iopub.execute_input":"2022-02-06T19:43:55.968287Z","iopub.status.idle":"2022-02-06T19:43:56.522897Z","shell.execute_reply.started":"2022-02-06T19:43:55.968252Z","shell.execute_reply":"2022-02-06T19:43:56.522244Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge\ndf2_text = pd.read_csv('./test.csv')\ndf2_label = pd.read_csv('./test_labels.csv')\n# join\ndf2 = pd.concat([df2_text, df2_label], axis=1)\n# select used column\ndf2 = df2[['comment_text', 'toxic', 'severe_toxic']]\n# remove row with value -1\ndf2 = df2[\n    (df2['toxic'] != -1)\n    &\n    (df2['severe_toxic'] != -1)\n]\n# set label with smoothing\nlabel = []\nfor row in df2.iterrows():\n    row = row[1]\n    if row['toxic'] == 0 and row['severe_toxic'] == 0:\n        label.append(0.05)\n    elif row['severe_toxic'] == 1:\n        label.append(0.95)\n    else:\n        label.append(0.5)\ndf2['label'] = label\n# under sample label 0.0\n# df2 = pd.concat([\n#     df2[\n#         (df2['label'] == 0.5)\n#         |\n#         (df2['label'] == 1.0)\n#     ],\n#     df2[df2['label'] == 0.0].sample(n=20000, random_state=SEED)\n# ])\ndf2","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:43:56.524249Z","iopub.execute_input":"2022-02-06T19:43:56.524485Z","iopub.status.idle":"2022-02-06T19:44:00.646328Z","shell.execute_reply.started":"2022-02-06T19:43:56.524452Z","shell.execute_reply":"2022-02-06T19:44:00.645689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification\ndf3_pub = pd.read_csv('/kaggle/input/jigsaw-unintended-bias-in-toxicity-classification/test_public_expanded.csv')\ndf3_prv = pd.read_csv('/kaggle/input/jigsaw-unintended-bias-in-toxicity-classification/test_private_expanded.csv')\n# join\ndf3 = pd.concat([df3_pub, df3_prv], axis=0)\n# select used column\ndf3 = df3[['comment_text', 'toxicity', 'severe_toxicity']]\n# undersample row with value 0.0 and smooth it to 0.05\ndf3 = pd.concat([\n    df3[df3['toxicity'] > 0],\n    (\n        df3.query('toxicity == 0.0')\n           .sample(n=13500, random_state=SEED)\n           .replace(0.0, 0.05)\n    )\n], axis=0)\n# severe_toxicity max score is 0.4, so multiply it by 2.5\ndf3['severe_toxicity'] = df3['severe_toxicity'] * 2.5\n# set label\nlabel = []\nfor row in df3.iterrows():\n    row = row[1]\n    if row['severe_toxicity'] > 0 and row['severe_toxicity'] > row['toxicity']:\n        label.append(row['severe_toxicity'])\n    else:\n        label.append(row['toxicity'] / 2)\ndf3['label'] = label\n# undersample label <= 0.4\n# df3 = pd.concat([\n#     df3[df3['label'] <= 0.1].sample(n=9000, random_state=SEED),\n#     df3[(df3['label'] > 0.1) & (df3['label'] <= 0.2)].sample(n=3000, random_state=SEED),\n#     df3[(df3['label'] > 0.2) & (df3['label'] <= 0.3)].sample(n=3000, random_state=SEED),\n#     df3[(df3['label'] > 0.3) & (df3['label'] <= 0.4)].sample(n=3000, random_state=SEED),\n#     df3[df3['label'] > 0.4]\n# ])\ndf3","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:44:00.647521Z","iopub.execute_input":"2022-02-06T19:44:00.649012Z","iopub.status.idle":"2022-02-06T19:44:06.891908Z","shell.execute_reply.started":"2022-02-06T19:44:00.648974Z","shell.execute_reply":"2022-02-06T19:44:06.891238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"X = (\n    df['text'].tolist() +\n    df2['comment_text'].tolist() +\n    df3['comment_text'].tolist()    \n)\ny = (\n    df['label'].tolist() +\n    df2['label'].tolist() +\n    df3['label'].tolist()\n)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:44:06.894678Z","iopub.execute_input":"2022-02-06T19:44:06.894990Z","iopub.status.idle":"2022-02-06T19:44:06.916951Z","shell.execute_reply.started":"2022-02-06T19:44:06.894944Z","shell.execute_reply":"2022-02-06T19:44:06.916162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!rm train.csv test.csv test_labels.csv\n\ndel m\ndel df_raw\ndel df\ndel df2_text\ndel df2_label\ndel df2\ndel df3_pub\ndel df3_prv\ndel df3\ndel label\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:44:06.918074Z","iopub.execute_input":"2022-02-06T19:44:06.918592Z","iopub.status.idle":"2022-02-06T19:44:07.979079Z","shell.execute_reply.started":"2022-02-06T19:44:06.918552Z","shell.execute_reply":"2022-02-06T19:44:07.978402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nclass TextDataset(torch.utils.data.Dataset):\n    def __init__(self, tokenizer, text, label=None):\n        self.tokenizer = tokenizer\n        self.text = text\n        self.label = label\n    \n    def __len__(self):\n        return len(self.text)\n    \n    def __getitem__(self, idx):\n        inputs = self.tokenizer(\n            self.text[idx],\n            truncation=True,\n            padding='max_length',\n            max_length=512\n        )\n        ids = inputs['input_ids']\n        mask = inputs['attention_mask']\n        token_type_ids = inputs[\"token_type_ids\"]\n        \n        if self.label is None:\n            return {\n                'ids': torch.tensor(ids, dtype=torch.long),\n                'mask': torch.tensor(mask, dtype=torch.long),\n                'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)\n            }            \n        else:\n            return {\n                'ids': torch.tensor(ids, dtype=torch.long),\n                'mask': torch.tensor(mask, dtype=torch.long),\n                'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n                'label': torch.tensor(self.label[idx], dtype=torch.float)\n            }\n\nclass JigsawDM(pl.LightningDataModule):\n    def __init__(self, tokenizer, X, y, batch_size):\n        super().__init__()\n        self.tokenizer = tokenizer\n        self.X = X\n        self.y = y\n        self.batch_size = batch_size\n\n    def setup(self, stage=None):\n        X_train, X_val, y_train, y_val = train_test_split(\n            self.X, self.y, test_size=VAL_PCT, random_state=SEED\n        )\n        \n        self.train_ds = TextDataset(self.tokenizer, X_train, y_train)\n        self.val_ds = TextDataset(self.tokenizer, X_val, y_val)\n\n        del self.X\n        del self.y\n        del X_train\n        del y_train\n        del X_val\n        del y_val\n        gc.collect()\n\n    def train_dataloader(self):\n        return torch.utils.data.DataLoader(self.train_ds, batch_size=self.batch_size, shuffle=True, num_workers=2, pin_memory=True)\n\n    def val_dataloader(self):\n        return torch.utils.data.DataLoader(self.val_ds, batch_size=self.batch_size, shuffle=False, num_workers=2, pin_memory=True)\n    \ndm = JigsawDM(tokenizer, X, y, batch_size=BATCH_SIZE)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:44:07.980599Z","iopub.execute_input":"2022-02-06T19:44:07.980846Z","iopub.status.idle":"2022-02-06T19:44:08.059747Z","shell.execute_reply.started":"2022-02-06T19:44:07.980812Z","shell.execute_reply":"2022-02-06T19:44:08.059107Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Train","metadata":{}},{"cell_type":"code","source":"from pytorch_lightning.callbacks import StochasticWeightAveraging, ModelCheckpoint\n\ntrainer = pl.Trainer(\n    gpus=1,\n    precision=16,\n    max_epochs=EPOCH,\n    default_root_dir='./trainer_cp',\n    log_every_n_steps=10,\n    callbacks=[\n        StochasticWeightAveraging(swa_epoch_start=1, annealing_epochs=1),\n        ModelCheckpoint(dirpath='./model_cp', filename='jigsaw-debertav3-{epoch:02d}-{val_loss:.4f}', monitor='val_loss', save_top_k=3)\n    ]\n)\n\ntrainer.fit(model, datamodule=dm)","metadata":{"execution":{"iopub.status.busy":"2022-02-06T19:44:08.060753Z","iopub.execute_input":"2022-02-06T19:44:08.061010Z"},"trusted":true},"execution_count":null,"outputs":[]}]}