{"cells":[{"metadata":{},"cell_type":"markdown","source":" As same as test set, I also created translated validation set. Even though test set is now translated to English, it looks still different from orignal English text. To evaluate your model performance, it makes more sense if you use translated validation set which is translated by same way as test set.  \n This kernel is also forked from [Abhishek's kernel](https://www.kaggle.com/abhishek/bert-multi-lingual-tpu-training-8-cores), please check it out!"},{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"993abe0b-2561-4abc-a6a2-b83d8dd4c846","_cell_guid":"16a3df23-8416-46b5-8661-c52345005b6d","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\n\nfrom tqdm import tqdm\nfrom collections import OrderedDict, namedtuple\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport joblib\n\nimport logging\nimport transformers\nfrom transformers import AdamW, get_linear_schedule_with_warmup, get_constant_schedule\nimport sys\nfrom sklearn import metrics, model_selection\n\nimport warnings\nimport torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\nclass AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\nclass BERTBaseUncased(nn.Module):\n    def __init__(self, bert_path):\n        super(BERTBaseUncased, self).__init__()\n        self.bert_path = bert_path\n        self.bert = transformers.BertModel.from_pretrained(self.bert_path)\n        self.bert_drop = nn.Dropout(0.3)\n        self.out = nn.Linear(768 * 2, 1)\n\n    def forward(\n            self,\n            ids,\n            mask,\n            token_type_ids\n    ):\n        o1, o2 = self.bert(\n            ids,\n            attention_mask=mask,\n            token_type_ids=token_type_ids)\n        \n        apool = torch.mean(o1, 1)\n        mpool, _ = torch.max(o1, 1)\n        cat = torch.cat((apool, mpool), 1)\n\n        bo = self.bert_drop(cat)\n        p2 = self.out(bo)\n        return p2\n\n\nclass BERTDatasetTraining:\n    def __init__(self, comment_text, targets, tokenizer, max_length):\n        self.comment_text = comment_text\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n        self.targets = targets\n\n    def __len__(self):\n        return len(self.comment_text)\n\n    def __getitem__(self, item):\n        comment_text = str(self.comment_text[item])\n        comment_text = \" \".join(comment_text.split())\n\n        inputs = self.tokenizer.encode_plus(\n            comment_text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_length,\n        )\n        ids = inputs[\"input_ids\"]\n        token_type_ids = inputs[\"token_type_ids\"]\n        mask = inputs[\"attention_mask\"]\n        \n        padding_length = self.max_length - len(ids)\n        \n        ids = ids + ([0] * padding_length)\n        mask = mask + ([0] * padding_length)\n        token_type_ids = token_type_ids + ([0] * padding_length)\n        \n        return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n            'targets': torch.tensor(self.targets[item], dtype=torch.float)\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mx = BERTBaseUncased(bert_path=\"../input/bert-base-multilingual-uncased/\")\ndf_train1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=[\"comment_text\", \"toxic\"]).fillna(\"none\")\ndf_train2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\", usecols=[\"comment_text\", \"toxic\"]).fillna(\"none\")\ndf_train_full = pd.concat([df_train1, df_train2], axis=0).reset_index(drop=True)\ndf_train = df_train_full.sample(frac=1).reset_index(drop=True).head(1000)\n\ndf_valid = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ndf_valid_en = pd.read_csv('../input/val-en-df/validation_en.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _run():\n    def loss_fn(outputs, targets):\n        return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))\n\n    def train_loop_fn(data_loader, model, optimizer, device, scheduler=None):\n        model.train()\n        for bi, d in enumerate(data_loader):\n            ids = d[\"ids\"]\n            mask = d[\"mask\"]\n            token_type_ids = d[\"token_type_ids\"]\n            targets = d[\"targets\"]\n\n            ids = ids.to(device, dtype=torch.long)\n            mask = mask.to(device, dtype=torch.long)\n            token_type_ids = token_type_ids.to(device, dtype=torch.long)\n            targets = targets.to(device, dtype=torch.float)\n\n            optimizer.zero_grad()\n            outputs = model(\n                ids=ids,\n                mask=mask,\n                token_type_ids=token_type_ids\n            )\n\n            loss = loss_fn(outputs, targets)\n            if bi % 10 == 0:\n                xm.master_print(f'bi={bi}, loss={loss}')\n\n            loss.backward()\n            xm.optimizer_step(optimizer)\n            if scheduler is not None:\n                scheduler.step()\n\n    def eval_loop_fn(data_loader, model, device):\n        model.eval()\n        fin_targets = []\n        fin_outputs = []\n        for bi, d in enumerate(data_loader):\n            ids = d[\"ids\"]\n            mask = d[\"mask\"]\n            token_type_ids = d[\"token_type_ids\"]\n            targets = d[\"targets\"]\n\n            ids = ids.to(device, dtype=torch.long)\n            mask = mask.to(device, dtype=torch.long)\n            token_type_ids = token_type_ids.to(device, dtype=torch.long)\n            targets = targets.to(device, dtype=torch.float)\n\n            outputs = model(\n                ids=ids,\n                mask=mask,\n                token_type_ids=token_type_ids\n            )\n\n            targets_np = targets.cpu().detach().numpy().tolist()\n            outputs_np = outputs.cpu().detach().numpy().tolist()\n            fin_targets.extend(targets_np)\n            fin_outputs.extend(outputs_np)    \n\n        return fin_outputs, fin_targets\n\n    \n    MAX_LEN = 192\n    TRAIN_BATCH_SIZE = 64\n    EPOCHS = 2\n\n    tokenizer = transformers.BertTokenizer.from_pretrained(\"../input/bert-base-multilingual-uncased/\", do_lower_case=True)\n\n    train_targets = df_train.toxic.values\n    valid_targets = df_valid.toxic.values\n\n    train_dataset = BERTDatasetTraining(\n        comment_text=df_train.comment_text.values,\n        targets=train_targets,\n        tokenizer=tokenizer,\n        max_length=MAX_LEN\n    )\n\n    train_sampler = torch.utils.data.distributed.DistributedSampler(\n          train_dataset,\n          num_replicas=xm.xrt_world_size(),\n          rank=xm.get_ordinal(),\n          shuffle=True)\n\n    train_data_loader = torch.utils.data.DataLoader(\n        train_dataset,\n        batch_size=TRAIN_BATCH_SIZE,\n        sampler=train_sampler,\n        drop_last=True,\n        num_workers=4\n    )\n\n    valid_dataset = BERTDatasetTraining(\n        comment_text=df_valid.comment_text.values,\n        targets=valid_targets,\n        tokenizer=tokenizer,\n        max_length=MAX_LEN\n    )\n\n    valid_sampler = torch.utils.data.distributed.DistributedSampler(\n          valid_dataset,\n          num_replicas=xm.xrt_world_size(),\n          rank=xm.get_ordinal(),\n          shuffle=False)\n\n    valid_data_loader = torch.utils.data.DataLoader(\n        valid_dataset,\n        batch_size=64,\n        sampler=valid_sampler,\n        drop_last=False,\n        num_workers=4\n    )\n    \n    #--------------------------------------------------------------------------------\n    # Here is english translated validation set\n    #--------------------------------------------------------------------------------\n    valid_en_dataset = BERTDatasetTraining(\n        comment_text=df_valid_en.comment_text_en.values,\n        targets=valid_targets,\n        tokenizer=tokenizer,\n        max_length=MAX_LEN\n    )\n\n    valid_en_sampler = torch.utils.data.distributed.DistributedSampler(\n          valid_en_dataset,\n          num_replicas=xm.xrt_world_size(),\n          rank=xm.get_ordinal(),\n          shuffle=False)\n\n    valid_en_data_loader = torch.utils.data.DataLoader(\n        valid_en_dataset,\n        batch_size=64,\n        sampler=valid_sampler,\n        drop_last=False,\n        num_workers=4\n    )\n\n    device = xm.xla_device()\n    model = mx.to(device)\n\n    param_optimizer = list(model.named_parameters())\n    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n    optimizer_grouped_parameters = [\n        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]\n\n    lr = 0.4 * 1e-5 * xm.xrt_world_size()\n    num_train_steps = int(len(train_dataset) / TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)\n    xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size()}')\n\n    optimizer = AdamW(optimizer_grouped_parameters, lr=lr)\n    scheduler = get_linear_schedule_with_warmup(\n        optimizer,\n        num_warmup_steps=0,\n        num_training_steps=num_train_steps\n    )\n\n    for epoch in range(EPOCHS):\n        para_loader = pl.ParallelLoader(train_data_loader, [device])\n        train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)\n\n        para_loader = pl.ParallelLoader(valid_data_loader, [device])\n        o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)\n        xm.save(model.state_dict(), \"model.bin\")\n        auc = metrics.roc_auc_score(np.array(t) >= 0.5, o)\n        xm.master_print(f'ORIGINAL VAL AUC = {auc}')\n        \n        # Check how your model perform for translated val set\n        para_loader = pl.ParallelLoader(valid_en_data_loader, [device])\n        o, t = eval_loop_fn(para_loader.per_device_loader(device), model, device)\n        auc = metrics.roc_auc_score(np.array(t) >= 0.5, o)\n        xm.master_print(f'TRANSLATED VAL AUC = {auc}')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Start training processes\ndef _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    a = _run()\n\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=1, start_method='fork')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}