{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport torch\nimport torch.nn as nn\nimport os\nfrom scipy import stats\nfrom tqdm import tqdm\nfrom collections import OrderedDict , namedtuple\nfrom torch.optim import lr_scheduler\nimport joblib\nimport logging \nimport transformers\nfrom transformers import AdamW , get_linear_schedule_with_warmup, get_constant_schedule\nimport sys\nfrom sklearn import metrics , model_selection\n\nimport warnings\nimport torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\n\nwarnings.filterwarnings('ignore')\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BERTModel(nn.Module):\n    def __init__(self , bert_path):\n        super(BERTModel , self).__init__()\n        self.bert_path = bert_path\n        self.bert = transformers.BertModel.from_pretrained(self.bert_path)\n        self.bert_drop = nn.Dropout(0.3)\n        self.out = nn.Linear ( 768*2 , 1 )\n        \n    def forward(self , ids , mask , token_type_ids):\n        o1 , o2 = self.bert ( ids , attention_mask = mask , token_type_ids = token_type_ids)\n        avgpool = torch.mean(o1 , 1)\n        maxpool, _ = torch.max(o1 ,1)\n        cat = torch.cat((avgpool,maxpool) , 1 )\n        bo = self.bert_drop(cat)\n        p2 = self.out(bo)\n        return p2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BERTDataset:\n    def __init__(self, comment_text, targets, tokenizer, max_length):\n        self.comment_text = comment_text\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n        self.targets = targets\n\n    def __len__(self):\n        return len(self.comment_text)\n\n    def __getitem__(self, item):\n        comment_text = str(self.comment_text[item])\n        comment_text = \" \".join(comment_text.split())\n\n        inputs = self.tokenizer.encode_plus(\n            comment_text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_length,\n        )\n        ids = inputs[\"input_ids\"]\n        token_type_ids = inputs[\"token_type_ids\"]\n        mask = inputs[\"attention_mask\"]\n        \n        padding_length = self.max_length - len(ids)\n        \n        ids = ids + ([0] * padding_length)\n        mask = mask + ([0] * padding_length)\n        token_type_ids = token_type_ids + ([0] * padding_length)\n        \n        return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n            'targets': torch.tensor(self.targets[item], dtype=torch.float)\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mx = BERTModel(bert_path=\"../input/bert-base-multilingual-uncased/\")\ndf_train1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=[\"comment_text\", \"toxic\"]).fillna(\"none\")\ndf_train2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\", usecols=[\"comment_text\", \"toxic\"]).fillna(\"none\")\ndf_train_full = pd.concat([df_train1, df_train2], axis=0).reset_index(drop=True)\ndf_train = df_train_full.sample(frac=1).reset_index(drop=True).head(200000)\n\ndf_valid = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/validation.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def loss_fn(outputs ,targets):\n    return nn.BCEWithLogitsLoss()(outputs , targets.view(-1,1))\n\ndef trainloop(data_loader , model , optimizer ,device , scheduler = None):\n    model.train()\n    for bi , d in enumerate(data_loader):\n        ids = d['ids']\n        targets = d['targets']\n        mask = d['mask']\n        token_type_ids = d['token_type_ids']\n        \n        ids=  ids.to(device , dtype = torch.long)\n        targets=  targets.to(device , dtype = torch.float)\n        mask=  mask.to(device , dtype = torch.long)\n        token_type_ids=  token_type_ids.to(device , dtype = torch.long)\n        \n        optimizer.zero_grad()\n        outputs = model(ids=ids,  mask=mask , token_type_ids = token_type_ids)\n        \n        loss=loss_fn(outputs ,targets)\n        if bi%10 ==0:\n            xm.master_print(f' Batch Index : {bi}  ||  Loss : {loss}')\n        \n                            \n               \n        loss.backward()\n        xm.optimizer_step(optimizer)\n        if scheduler is not None:\n            scheduler.step()\n            \n            \ndef evalloop(data_loader , model , device):\n    \n    model.eval()\n    fin_targets = []\n    fin_outputs = []\n    for bi, d in enumerate(data_loader):\n        ids = d[\"ids\"]\n        mask = d[\"mask\"]\n        token_type_ids = d[\"token_type_ids\"]\n        targets = d[\"targets\"]\n\n        ids = ids.to(device, dtype=torch.long)\n        mask = mask.to(device, dtype=torch.long)\n        token_type_ids = token_type_ids.to(device, dtype=torch.long)\n        targets = targets.to(device, dtype=torch.float)\n\n        outputs = model(\n                ids=ids,\n                mask=mask,\n                token_type_ids=token_type_ids\n            )\n\n        targets_np = targets.cpu().detach().numpy().tolist()\n        outputs_np = outputs.cpu().detach().numpy().tolist()\n        fin_targets.extend(targets_np)\n        fin_outputs.extend(outputs_np)\n                \n\n    return fin_outputs, fin_targets       \n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _run():\n    \n    MAX_LEN = 192\n    TRAIN_BATCH_SIZE = 64\n    VAL_BATCH_SIZE = 64\n    EPOCHS = 5\n    \n    tokenizer = transformers.BertTokenizer.from_pretrained('../input/bert-base-multilingual-uncased/' , do_lower_case = True)\n    \n    train_targets = df_train.toxic.values\n    valid_targets = df_valid.toxic.values\n    \n    train_dataset = BERTDataset(comment_text = df_train.comment_text.values,\n                                targets = train_targets,\n                                tokenizer= tokenizer,\n                                max_length = MAX_LEN\n                               )\n    \n    train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset,\n                                                                    num_replicas = xm.xrt_world_size(),\n                                                                    rank = xm.get_ordinal(),\n                                                                    shuffle= True)\n    \n    train_dataloader = torch.utils.data.DataLoader(train_dataset,\n                                                   batch_size=  TRAIN_BATCH_SIZE,\n                                                   sampler = train_sampler,\n                                                   drop_last = True,\n                                                   num_workers = 4)\n    \n    \n    val_dataset = BERTDataset(comment_text = df_valid.comment_text.values,\n                                targets = valid_targets,\n                                tokenizer= tokenizer,\n                                max_length = MAX_LEN\n                               )\n    \n    val_sampler = torch.utils.data.distributed.DistributedSampler(val_dataset,\n                                                                    num_replicas = xm.xrt_world_size(),\n                                                                    rank = xm.get_ordinal(),\n                                                                    shuffle= False)\n    \n    val_dataloader = torch.utils.data.DataLoader(val_dataset,\n                                                   batch_size=  VAL_BATCH_SIZE,\n                                                   sampler = val_sampler,\n                                                   drop_last = False,\n                                                   num_workers = 4)\n    \n    device = xm.xla_device()\n    model = mx.to(device)\n    \n    \n    param_optimizer = list(model.named_parameters())\n    no_decay = ['bias' , 'LayerNorm.bias' , 'LayerNorm.weight']\n    optimizer_grouped_params = [\n        {'params' : [p for n , p in param_optimizer if not any (nd in n for nd in no_decay)] , 'weight_decay' : 0.001},\n        {'params' : [p for n , p in param_optimizer if any (nd in n for nd in no_decay)] , 'weight_decay' : 0.0}\n       ]\n    \n    lr = 0.4 * 1e-5 * xm.xrt_world_size()\n    num_train_steps = int(len(train_dataset) / TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS )\n    xm.master_print(f' NUM_TRAIN_STEPS : {num_train_steps} || XRT_WORLD_SIZE : { xm.xrt_world_size()}')\n    \n    \n    optimizer = AdamW(optimizer_grouped_params,lr=lr)\n    scheduler = get_linear_schedule_with_warmup(optimizer,  num_warmup_steps = 0 , num_training_steps = num_train_steps)\n    \n    \n    for epoch in range(EPOCHS):\n        para_loader = pl.ParallelLoader(train_dataloader , [device])\n        trainloop(para_loader.per_device_loader(device) , model, optimizer,  device,  scheduler=scheduler)\n        para_loader = pl.ParallelLoader(val_dataloader , [device])\n        o,t= evalloop(para_loader.per_device_loader(device) , model , device )\n        xm.save(model.state_dict ,'bertmodel.bin')\n        auc = metrics.roc_auc_score(np.array(t) >= 0.5 , o )\n        xm.master_print(f'ROC-AUC Score : {auc}')\n        \n                        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _multiprocessing_train(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    a = _run()\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"FLAGS = {}\nxmp.spawn(_multiprocessing_train , args= (FLAGS,), nprocs = 8 , start_method = 'fork' )","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}