{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# import torch_xla.utils.serialization as xser","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\nimport os\nimport torch\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\n\nfrom tqdm import tqdm\nfrom collections import OrderedDict, namedtuple\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport joblib\n\nimport logging\nimport transformers\nfrom transformers import AdamW, get_linear_schedule_with_warmup, get_constant_schedule, BertModel, XLMRobertaModel, XLMRobertaTokenizer, DistilBertModel, DistilBertTokenizer\nimport sys\nfrom sklearn import metrics, model_selection\n\nimport warnings\nimport torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\nimport warnings\nimport time\n\nwarnings.filterwarnings(\"ignore\")\n\nclass AverageMeter:\n    \"\"\"\n    Computes and stores the average and current value\n    \"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count\n\n\ndef onehot(size, target):\n    vec = torch.zeros(size, dtype=torch.float32)\n    vec[target] = 1.\n    return vec\n        \nclass BERTDatasetTraining(torch.utils.data.TensorDataset):\n    def __init__(self, comment_text, targets, tokenizer, idxs=None, max_length=200, test=False):\n        self.comment_text = comment_text\n        self.test = test\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n        self.targets = targets\n        self.idxs = idxs\n\n    def get_tokens(self, text):\n        encoded = self.tokenizer.encode_plus(\n            text, \n            add_special_tokens=True, \n            max_length=self.max_length, \n            pad_to_max_length=True\n        )\n        return encoded['input_ids'], encoded['attention_mask']\n\n    def __getitem__(self, item):\n    \n        text = self.comment_text[item]\n\n        encoded = self.get_tokens(text)\n\n        if not self.test:\n            return torch.tensor(encoded[0],dtype=torch.long), torch.tensor(encoded[1],dtype=torch.long), torch.tensor(self.targets[item], dtype=torch.float)\n        else:\n            return torch.tensor(self.idxs[item]), torch.tensor(encoded[0],dtype=torch.long), torch.tensor(encoded[1],dtype=torch.long)          \n\n    def __len__(self):\n        return len(self.targets)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean(text):\n    if re.findall(r'\\[\\'\\'\\]',text):\n        return ''\n    else:\n        retext = re.split(',',text)\n        clean_text = []\n        retext[0] = retext[0][1:]\n        retext[-1] = retext[-1][:-1]\n        for i in range(len(retext)):\n            retext[i] = \" \".join(retext[i].split())\n            retext[i] = retext[i][1:-1]\n        return '.'.join(retext)\n\ndef clean_test(text):\n    if re.findall(r'\\[\\'\\'\\]',text):\n        return '[UNK]'\n    else:\n        retext = re.split(',',text)\n        clean_text = []\n        retext[0] = retext[0][1:]\n        retext[-1] = retext[-1][:-1]\n        for i in range(len(retext)):\n            retext[i] = \" \".join(retext[i].split())\n            retext[i] = retext[i][1:-1]\n        return '.'.join(retext)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class ToxicSimpleNNModel(nn.Module):\n\n    def __init__(self):\n        super(ToxicSimpleNNModel, self).__init__()\n        self.encoder = BertModel.from_pretrained(\"../input/bert-base-multilingual-uncased/\")\n#         self.encoder = DistilBertModel.from_pretrained(\"distilbert-base-multilingual-cased\")\n#         self.encoder = XLMRobertaModel.from_pretrained(\"../input/bert-base-multilingual-uncased/\")\n        self.dropout = nn.Dropout(0.3)\n        self.linear_1 = nn.Linear(\n            in_features=self.encoder.pooler.dense.out_features*2,\n            out_features=self.encoder.pooler.dense.out_features*2,\n        )\n        self.linear_2 = nn.Linear(\n            in_features=self.encoder.pooler.dense.out_features*2,\n            out_features=self.encoder.pooler.dense.out_features,\n        )\n        self.linear_3 = nn.Linear(\n            in_features=self.encoder.pooler.dense.out_features,\n            out_features=1,\n        )\n\n    def forward(self, input_ids, attention_mask):\n        # bs, seq_length = input_ids.shape\n        seq_x = self.encoder(input_ids=input_ids, attention_mask=attention_mask)\n        apool = torch.mean(seq_x[0], 1)\n        mpool, _ = torch.max(seq_x[0], 1)\n        x = torch.cat((apool, mpool), 1)\n        x = self.dropout(x)\n        x = self.linear_1(x)\n        x = self.dropout(x)\n        x = self.linear_2(x)\n        x = self.dropout(x)\n        return self.linear_3(x)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# mx = BERTBaseUncased(bert_path=\"../input/bert-base-multilingual-uncased/\")\nmx=ToxicSimpleNNModel()\n\ndf_train1 = pd.read_csv(\"../input/jigsaw-dataset/train_sent_prep.csv\", usecols=[\"comment_text\", \"toxic\"],converters={'comment_text': lambda x: clean(x)}).fillna(\"none\")\ndf_train1 = df_train1[df_train1.comment_text!='']\ndf_train_full = df_train1.reset_index(drop=True)\ntrain = df_train_full.sample(frac=1).reset_index(drop=True)\ndel df_train1,df_train_full\n\ndf_valid = pd.read_csv(\"../input/jigsaw-dataset/validation_sent_pr.csv\", usecols=[\"comment_text\", \"toxic\"], converters={'comment_text': lambda x: clean(x)})\nvalid = df_valid[df_valid.comment_text!=''].reset_index(drop=True)\ndel df_valid\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test = pd.read_csv(\"../input/jigsaw-dataset/test_sent_pr.csv\", usecols=[\"content\"], converters={'content': lambda x: clean_test(x)})\ndf_test['comment_text'] = df_test.content\ntest = df_test[['comment_text']].reset_index(drop=True)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class RocAucMeter(object):\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.y_true = np.array([0,1])\n        self.y_pred = np.array([0.5,0.5])\n        self.score = 0\n\n    def update(self, y_true, y_pred):\n#         y_true = y_true.cpu().numpy().argmax(axis=1)\n        y_true = y_true.cpu().numpy()\n        y_pred = torch.sigmoid(y_pred).cpu().detach().numpy()[:,0]\n#         y_pred = nn.functional.softmax(y_pred, dim=1).data.cpu().numpy()[:,1]\n        self.y_true = np.hstack((self.y_true, y_true))\n        self.y_pred = np.hstack((self.y_pred, y_pred))\n        self.score = metrics.roc_auc_score(self.y_true, self.y_pred, labels=np.array([0, 1]))\n    \n    @property\n    def avg(self):\n        return self.score\n\nclass AverageMeter(object):\n    \"\"\"Computes and stores the average and current value\"\"\"\n    def __init__(self):\n        self.reset()\n\n    def reset(self):\n        self.val = 0\n        self.avg = 0\n        self.sum = 0\n        self.count = 0\n\n    def update(self, val, n=1):\n        self.val = val\n        self.sum += val * n\n        self.count += n\n        self.avg = self.sum / self.count","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def loss_fn(outputs, targets):\n    return nn.BCEWithLogitsLoss()(outputs, targets.view(-1, 1))\n\n# class LabelSmoothing(nn.Module):\n#     def __init__(self, smoothing = 0.1):\n#         super(LabelSmoothing, self).__init__()\n#         self.confidence = 1.0 - smoothing\n#         self.smoothing = smoothing\n\n#     def forward(self, x, target):\n#         if self.training:\n#             x = x.float()\n#             target = target.float()\n#             logprobs = torch.nn.functional.log_softmax(x, dim = -1)\n#             nll_loss = -logprobs * target\n#             nll_loss = nll_loss.sum(-1)\n#             smooth_loss = -logprobs.mean(dim=-1)\n#             loss = self.confidence * nll_loss + self.smoothing * smooth_loss\n#             return loss.mean()\n#         else:\n#             return torch.nn.functional.cross_entropy(x, target)\n        \n# def loss_fn(outputs, targets):\n#     return LabelSmoothing()(outputs, targets)\n    \ndef train_loop_fn(loader,model,optimizer,device,scheduler):\n    tracker = xm.RateTracker()\n    # cb = callback()\n    losses = AverageMeter()\n    auc = RocAucMeter()\n    # start_time = time.time()\n\n\n    gradient_accumulation_steps = 1\n    model.train()\n    for step, data in enumerate(loader):\n        input_ids, input_masks, labels = data\n        pred = model(input_ids = input_ids,\n                    attention_mask = input_masks\n                  )\n        loss = loss_fn(pred, labels.float())\n        loss.backward()\n        loss = loss.detach().item()\n\n        auc.update(labels, pred)\n        losses.update(loss, input_ids.size(0))\n        if (step + 1) % gradient_accumulation_steps == 0:\n          # Calling the step function on an Optimizer makes an update to its parameters\n            xm.optimizer_step(optimizer)\n            scheduler.step()\n            optimizer.zero_grad()\n\n#         tracker.add(FLAGS['batch_size'])\n        if (step+1) % 20 == 0:\n            xm.master_print('[xla:{}]({}) Rate={:.2f} Loss={:.4f} AUC={:.4f} GlobalRate={:.2f} Time={}'.format(\n                xm.get_ordinal(), step+1, tracker.rate(), losses.avg, auc.avg,\n                tracker.global_rate(), time.asctime()))\n    del loss\n#     del losses\n#     del outputs\n    del input_ids\n    del labels\n  \n    return losses, auc\n\ndef valid_loop_fn(loader,model,device): \n    losses = AverageMeter()\n    final_scores = RocAucMeter()\n\n    model.eval()\n    with torch.no_grad():\n        for j, data in enumerate(loader):\n\n          # get the inputs\n            input_ids, input_masks, labels = data\n            pred = model(input_ids = input_ids.long(),\n                       attention_mask = input_masks\n                      )\n\n            loss_val = loss_fn(pred, labels.float())\n            final_scores.update(labels, pred)\n            losses.update(loss_val.detach().item(), input_ids.size(0))\n            del loss_val,pred,input_ids,input_masks,labels\n\n    return losses, final_scores\n\n# def valid_loop_fn(loader,model,device):\n#     losses = AverageMeter()\n#     final_scores = RocAucMeter()\n#     fin_targets = []\n#     fin_outputs = []\n\n#     model.eval()\n#     with torch.no_grad():\n#         for j, data in enumerate(loader):\n\n#           # get the inputs\n#             input_ids, input_masks, labels = data\n#             pred = model(input_ids = input_ids.long(),\n#                        attention_mask = input_masks\n#                       )\n\n#             loss_val = loss_fn(pred, labels.float())\n#             final_scores.update(labels, pred)\n#             losses.update(loss_val.detach().item(), input_ids.size(0))\n#             targets_np = labels.cpu().detach().numpy().tolist()\n#             outputs_np = pred.cpu().detach().numpy()[:,1].tolist()\n#             fin_targets.extend(targets_np)\n#             fin_outputs.extend(outputs_np)\n#             del loss_val,pred,input_ids,input_masks,labels\n\n#     return losses, final_scores, fin_outputs, fin_targets","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def test_loop_fn(loader,model,device): \n\n    model.eval()\n    result = {'id': [], 'toxic': []}\n    with torch.no_grad():\n        for j, data in enumerate(loader):\n        \n          # get the inputs\n            idxs, input_ids, input_masks = data\n            pred = model(input_ids = input_ids.long(),\n                       attention_mask = input_masks\n                      )\n            y_pred = torch.sigmoid(pred).cpu().detach().numpy()[:,0]\n            result['id'].extend(idxs.cpu().numpy())\n            result['toxic'].extend(y_pred)\n            del pred,input_ids,input_masks,idxs\n\n    return result","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mx.load_state_dict(torch.load('model_0.pt'))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!mkdir ./node_submissions/","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import datetime\nimport glob\nimport random","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _run():\n    \n    MAX_LEN = 192\n    TRAIN_BATCH_SIZE = 64\n    EPOCHS = 1\n\n#     tokenizer = transformers.BertTokenizer.from_pretrained(\"../input/bertbase-multilingual-cased/\", do_lower_case=True)\n    tokenizer = transformers.BertTokenizer.from_pretrained(\"bert-base-multilingual-uncased\", do_lower_case=True)\n    \n#     tokenizer = transformers.XLMRobertaTokenizer.from_pretrained(\"../input/xlm-roberta-base/\", do_lower_case=True)\n\n    count_proc = xm.xrt_world_size()\n    num_proc = xm.get_ordinal()\n    size_tr_dataset = round(len(train)/count_proc)\n    size_val_dataset = round(len(valid)/count_proc)\n    size_test_dataset = round(len(test)/count_proc)\n    print('Всего ядер {}'.format(count_proc))\n\n    train_data_loader = torch.utils.data.DataLoader(BERTDatasetTraining(\n        comment_text = train[num_proc*size_tr_dataset:(num_proc+1)*size_tr_dataset].comment_text.values,\n        targets=train[num_proc*size_tr_dataset:(num_proc+1)*size_tr_dataset].toxic.values, max_length=MAX_LEN, tokenizer=tokenizer),\n                                                    batch_size=TRAIN_BATCH_SIZE, shuffle=False,\n                                                    num_workers = 4, drop_last=False)\n        \n    valid_data_loader = torch.utils.data.DataLoader(BERTDatasetTraining(\n        comment_text=valid[num_proc*size_val_dataset:(num_proc+1)*size_val_dataset].comment_text.values,\n        targets=valid[num_proc*size_val_dataset:(num_proc+1)*size_val_dataset].toxic.values, max_length=MAX_LEN, tokenizer=tokenizer),\n                                                    batch_size=TRAIN_BATCH_SIZE, shuffle=False,\n                                                    num_workers = 4, drop_last=False)\n    \n    test_data_loader = torch.utils.data.DataLoader(BERTDatasetTraining(\n        comment_text=valid[num_proc*size_test_dataset:(num_proc+1)*size_test_dataset].comment_text.values,\n        targets=valid[num_proc*size_test_dataset:(num_proc+1)*size_test_dataset].toxic.values,\n        idxs=test[num_proc*size_test_dataset:(num_proc+1)*size_test_dataset].index.values, max_length=MAX_LEN, tokenizer=tokenizer, test=True),\n                                                    batch_size=TRAIN_BATCH_SIZE, shuffle=False,\n                                                    num_workers = 4, drop_last=False)\n\n    device = xm.xla_device()\n    model = mx.to(device)\n\n    param_optimizer = list(model.named_parameters())\n#     no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n#     optimizer_grouped_parameters = [\n#         {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n#         {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]\n\n    lr = 0.4 * 1e-5 * xm.xrt_world_size()\n    xm.master_print('Количество батчей {}'.format(len(train_data_loader)))\n    num_train_steps = int(len(train_data_loader) * EPOCHS)\n#     num_train_steps = int(len(train_dataset) / TRAIN_BATCH_SIZE / xm.xrt_world_size() * EPOCHS)\n    warmup_proportion = 0.01\n    num_warmup_steps = round(num_train_steps * warmup_proportion)\n    xm.master_print(f'num_train_steps = {num_train_steps}, world_size={xm.xrt_world_size()},  num_warmup_steps={num_warmup_steps}')\n\n#     optimizer = AdamW(optimizer_grouped_parameters, lr=lr)\n    optimizer = AdamW(model.parameters(), lr=lr)\n    scheduler = get_linear_schedule_with_warmup(\n        optimizer,\n        num_warmup_steps=num_warmup_steps,\n        num_training_steps=num_train_steps\n    )\n    \n    best_valid_loss = float(\"Inf\")\n\n    for epoch in range(EPOCHS):\n        start_time = time.time()\n        xm.master_print(\"Start training epoch {}\".format(epoch))\n        para_loader = pl.ParallelLoader(train_data_loader, [device])\n        train_loss, train_score = train_loop_fn(para_loader.per_device_loader(device), model, optimizer, device, scheduler=scheduler)\n        del para_loader\n        xm.master_print(f'[RESULT]: Train. Epoch: {epoch}, loss: {train_loss.avg:.5f}, final_score: {train_score.avg:.5f}, time: {(time.time() - start_time):.5f}')\n\n        para_loader = pl.ParallelLoader(valid_data_loader, [device])\n        valid_loss, valid_score= valid_loop_fn(para_loader.per_device_loader(device), model, device)\n#         o, t = valid_loop_fn(para_loader.per_device_loader(device), model, device)\n#         auc = metrics.roc_auc_score(np.array(t), o)\n\n#         gc.collect()\n        xm.master_print('Epoch {}/{} \\t loss={:.4f}\\t val_loss={:.4f} \\t train_score={:.4f}\\t val_score={:.4f}\\t time={:.2f}s'\n                        .format(epoch+1, EPOCHS, train_loss.avg, valid_loss.avg, train_score.avg, valid_score.avg, (time.time() - start_time)))\n\n        del para_loader\n        print(f'[xla:{xm.get_ordinal()}] AUC = {valid_score.avg}')\n        \n        def reduce_fn(vals):\n            return sum(vals) / len(vals)\n\n        avgloss = xm.mesh_reduce('auc_reduce', valid_loss.avg, reduce_fn)\n        auc = xm.mesh_reduce('auc_reduce', valid_score.avg, reduce_fn)\n        xm.master_print(f'AUC AVG = {auc}')\n        \n        xm.save(model.state_dict(), \"model_{}.pt\".format(epoch))\n        if best_valid_loss > avgloss:\n            best_valid_loss = avgloss\n            xm.save(model.state_dict(), \"model.pt\")\n            para_loader = pl.ParallelLoader(test_data_loader, [device])\n            result = test_loop_fn(para_loader.per_device_loader(device), model, device)\n            result = pd.DataFrame(result)\n#             result.to_csv('submission.csv', index=False)\n            node_count = len(glob.glob('node_submissions/*.csv'))\n            result.to_csv(f'node_submissions/submission_{node_count}_{xm.get_ordinal()}_{random.random()}.csv', index=False)\n            del para_loader,result\n        del train_loss, train_score, valid_loss, valid_score\n#         del train_loss, train_score\n    para_loader = pl.ParallelLoader(train_data_loader, [device])\n    valid_loss, valid_score= valid_loop_fn(para_loader.per_device_loader(device), model,  optimizer, device, scheduler=scheduler)\n    xm.master_print('val_loss={:.4f} \\t  val_score={:.4f}\\t'\n                    .format(valid_loss.avg, valid_score.avg))\n    del para_loader\n    result = test_loop_fn(para_loader.per_device_loader(device), model, device)\n    result = pd.DataFrame(result)\n#     result.to_csv('submission.csv', index=False)\n    node_count = len(glob('node_submissions/*.csv'))\n    result.to_csv(f'node_submissions/submission_{node_count}_{xm.get_ordinal()}_{random.random()}.csv', index=False)\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    a = _run()\n\nFLAGS={}\nxmp.spawn(_mp_fn, args=(FLAGS,), nprocs=8, start_method='fork')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission = pd.concat([pd.read_csv(path) for path in glob.glob('node_submissions/*.csv')]).groupby('id').mean()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}