{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pathlib\nfrom typing import Any, Callable, Dict, Tuple, List\n\nimport numpy as np\nimport pandas as pd\nimport pytorch_lightning as pl\nfrom pytorch_lightning.loggers import WandbLogger\nimport torch\nimport torch.nn as nn\nimport torch.nn.functional as F\nfrom torch.utils.data import Dataset, DataLoader\nfrom tqdm.auto import tqdm\nfrom transformers import AutoModel, AutoTokenizer\n\n# Wandb login:\nfrom kaggle_secrets import UserSecretsClient\nimport wandb\nuser_secrets = UserSecretsClient()\nsecret_value = user_secrets.get_secret(\"wandb_api_key\")\nwandb.login(key=secret_value)\n\ndevice = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-25T04:36:43.894363Z","iopub.execute_input":"2021-12-25T04:36:43.894798Z","iopub.status.idle":"2021-12-25T04:36:44.289458Z","shell.execute_reply.started":"2021-12-25T04:36:43.89474Z","shell.execute_reply":"2021-12-25T04:36:44.284117Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Kudos to [this Kaggle kernel](https://www.kaggle.com/xhlulu/jigsaw-tpu-xlm-roberta).","metadata":{}},{"cell_type":"code","source":"ROOT_PATH = pathlib.Path(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\")\nMODEL = \"distilbert-base-multilingual-cased\"\nBATCH_SIZE = 32\nEPOCHS = 1\nLR = 1e-5\nMAX_DOC_LENGTH = 256","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:20:22.762457Z","iopub.execute_input":"2021-12-25T04:20:22.762865Z","iopub.status.idle":"2021-12-25T04:20:22.767724Z","shell.execute_reply.started":"2021-12-25T04:20:22.762823Z","shell.execute_reply":"2021-12-25T04:20:22.766842Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(ROOT_PATH / \"jigsaw-toxic-comment-train.csv\")\nvalid_df = pd.read_csv(ROOT_PATH / \"validation.csv\")\ntest_df = pd.read_csv(ROOT_PATH / \"test.csv\").rename(columns={\"content\": \"comment_text\"})\ntrain_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:20:30.916163Z","iopub.execute_input":"2021-12-25T04:20:30.916533Z","iopub.status.idle":"2021-12-25T04:20:34.501599Z","shell.execute_reply.started":"2021-12-25T04:20:30.9165Z","shell.execute_reply":"2021-12-25T04:20:34.500781Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df.sample(5)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:20:34.503162Z","iopub.execute_input":"2021-12-25T04:20:34.503551Z","iopub.status.idle":"2021-12-25T04:20:34.521893Z","shell.execute_reply.started":"2021-12-25T04:20:34.503513Z","shell.execute_reply":"2021-12-25T04:20:34.520945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"toxic\"].mean(), valid_df[\"toxic\"].mean()","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:20:39.536012Z","iopub.execute_input":"2021-12-25T04:20:39.536401Z","iopub.status.idle":"2021-12-25T04:20:39.543951Z","shell.execute_reply.started":"2021-12-25T04:20:39.536366Z","shell.execute_reply":"2021-12-25T04:20:39.543033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Data Preparation for PyTorch","metadata":{}},{"cell_type":"code","source":"class TestCommentsData(Dataset):\n    def __init__(self, df: pd.DataFrame):\n        self.comments = df[\"comment_text\"].values\n        \n    def __len__(self) -> int:\n        return len(self.comments)\n    \n    def __getitem__(self, idx):\n        return self.comments[idx]\n\nclass CommentsData(TestCommentsData):\n    def __init__(self, df: pd.DataFrame):\n        super().__init__(df)\n        self.toxic = df[\"toxic\"].values\n    \n    def __getitem__(self, idx):\n        return self.comments[idx], self.toxic[idx]\n    \ntrain_ds = CommentsData(train_df)\nvalid_ds = CommentsData(valid_df)\ntest_ds = TestCommentsData(test_df)\n\ntrain_dl = DataLoader(train_ds, batch_size=BATCH_SIZE, shuffle=True, drop_last=True, num_workers=4, pin_memory=True)\nvalid_dl = DataLoader(valid_ds, batch_size=BATCH_SIZE, shuffle=False, drop_last=False, num_workers=4, pin_memory=True)\ntest_dl = DataLoader(test_ds, batch_size=BATCH_SIZE, shuffle=False, drop_last=False, num_workers=4, pin_memory=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:32:33.652903Z","iopub.execute_input":"2021-12-25T04:32:33.653276Z","iopub.status.idle":"2021-12-25T04:32:33.665648Z","shell.execute_reply.started":"2021-12-25T04:32:33.653241Z","shell.execute_reply":"2021-12-25T04:32:33.664758Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, y = next(iter(train_dl))\nlen(x), y.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:32:37.548157Z","iopub.execute_input":"2021-12-25T04:32:37.548497Z","iopub.status.idle":"2021-12-25T04:32:37.895628Z","shell.execute_reply.started":"2021-12-25T04:32:37.548464Z","shell.execute_reply":"2021-12-25T04:32:37.89381Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model","metadata":{}},{"cell_type":"code","source":"class Model(nn.Module):\n    def __init__(self):\n        super().__init__()\n        self.base = AutoModel.from_pretrained(MODEL)\n        self.head = nn.Linear(self.base.config.dim, 1)\n        \n    def forward(self, x: Dict[str, torch.LongTensor]) -> torch.FloatTensor:\n        sequence_output = self.base(**x).last_hidden_state # shape of BS x (SEQ_LEN + 1) x 768\n        cls_token = sequence_output[:, 0, :]\n        return self.head(cls_token)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:33:01.700362Z","iopub.execute_input":"2021-12-25T04:33:01.70072Z","iopub.status.idle":"2021-12-25T04:33:01.712534Z","shell.execute_reply.started":"2021-12-25T04:33:01.700683Z","shell.execute_reply":"2021-12-25T04:33:01.711403Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class LightningModel(pl.LightningModule):\n    def __init__(self, model: nn.Module, tokenizer, loss_fn: Callable, lr: float, thresh: float=0.5):\n        super().__init__()\n        self.model = model\n        self.tokenizer = tokenizer\n        self.loss_fn = loss_fn\n        self.lr = lr\n        self.thresh = thresh\n        \n    def common_step(self, batch):\n        x, y = batch\n        tokenized_x = self.tokenizer(\n            x,\n            max_length=MAX_DOC_LENGTH,\n            truncation=True,\n            padding=True,\n            return_tensors=\"pt\",\n        )\n        tokenized_x_device = {k: v.to(self.device) for k, v in tokenized_x.items()}\n        logits = self.model(tokenized_x_device).squeeze()\n        loss = self.loss_fn(logits, y.float())\n        probabilities = torch.sigmoid(logits)\n        y_pred = (probabilities > self.thresh).long()\n        accuracy = (y_pred == y).float().mean()\n\n        return loss, accuracy\n        \n    def training_step(self, batch: Tuple[torch.FloatTensor, torch.LongTensor], *args: List[Any]):\n        loss, accuracy = self.common_step(batch)\n        self.log(\"training_loss\", loss, on_step=True, on_epoch=True)\n        self.log(\"training_accuracy\", accuracy, on_step=True, on_epoch=True)\n        \n        return loss\n        \n#     def on_epoch_end(self, *args):\n#         if self.current_epoch == 0:\n#             for p in self.model.base.parameters():\n#                 p.requires_grad = True\n        \n    def validation_step(self, batch: Tuple[torch.FloatTensor, torch.LongTensor], *args: List[Any]):\n        loss, accuracy = self.common_step(batch)\n        self.log(\"validation_loss\", loss, on_step=False, on_epoch=True)\n        self.log(\"validation_accuracy\", accuracy, on_step=False, on_epoch=True)\n        \n    def configure_optimizers(self):\n        return torch.optim.Adam(self.model.parameters(), lr=self.lr)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:33:02.587248Z","iopub.execute_input":"2021-12-25T04:33:02.587582Z","iopub.status.idle":"2021-12-25T04:33:02.600756Z","shell.execute_reply.started":"2021-12-25T04:33:02.58755Z","shell.execute_reply":"2021-12-25T04:33:02.599646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir -p /kaggle/working/logs\nmodel = Model()\ntokenizer = AutoTokenizer.from_pretrained(MODEL)\nloss_fn = nn.BCEWithLogitsLoss()\nlightning_model = LightningModel(model, tokenizer, loss_fn, LR)\n\nlogger = WandbLogger(\"toxic comments - pt\", \"/kaggle/working/logs/\", project=\"Toxic Comments\")\ntrainer = pl.Trainer(\n    max_epochs=EPOCHS,\n    gpus=torch.cuda.device_count(),\n    gradient_clip_val=1.0,\n    logger=logger,\n    precision=16,\n)\ntrainer.fit(lightning_model, train_dl, valid_dl)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:33:03.826101Z","iopub.execute_input":"2021-12-25T04:33:03.82645Z","iopub.status.idle":"2021-12-25T04:36:40.080022Z","shell.execute_reply.started":"2021-12-25T04:33:03.826418Z","shell.execute_reply":"2021-12-25T04:36:40.079138Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Model\n\n- Transformers: https://jalammar.github.io/illustrated-transformer/\n- BERT: https://jalammar.github.io/a-visual-guide-to-using-bert-for-the-first-time/","metadata":{}},{"cell_type":"markdown","source":"## Submission","metadata":{}},{"cell_type":"code","source":"with torch.no_grad():\n    model = model.eval().to(device)\n    y_preds = []\n    for x in tqdm(test_dl):\n        tokenized_x = tokenizer(\n            x,\n            max_length=MAX_DOC_LENGTH,\n            truncation=True,\n            padding=True,\n            return_tensors=\"pt\",\n        )\n        tokenized_x_device = {k: v.to(device) for k, v in tokenized_x.items()}\n        logits = model(tokenized_x_device).squeeze()\n        probabilities = torch.sigmoid(logits)\n        y_preds.append(probabilities.cpu())\n\nsub = pd.read_csv(ROOT_PATH / \"sample_submission.csv\")\nsub['toxic'] = torch.cat(y_preds).numpy()\nsub.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-12-25T04:38:26.079388Z","iopub.execute_input":"2021-12-25T04:38:26.079793Z","iopub.status.idle":"2021-12-25T04:38:44.67836Z","shell.execute_reply.started":"2021-12-25T04:38:26.079741Z","shell.execute_reply":"2021-12-25T04:38:44.676832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}