{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport os.path\nimport os\nimport sys\nfrom math import exp\nfrom sklearn import metrics\n\nimport torch\nimport torch.nn as nn\nfrom torch.nn.utils.clip_grad import clip_grad_norm_\nfrom torch.optim import Adam\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.distributed import DistributedSampler\nfrom transformers import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"home_dir = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\"\ndf = pd.read_csv(os.path.join(home_dir, \"jigsaw-toxic-comment-train-processed-seqlen128.csv\"))\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df.head(200000)\ndf_val = pd.read_csv(os.path.join(home_dir, \"validation-processed-seqlen128.csv\"))\ndf_test = pd.read_csv(os.path.join(home_dir, \"test-processed-seqlen128.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[\"toxic\"].hist()\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 64","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def str_to_t(s):\n    return torch.tensor(np.array(s[1:-1].split(',')).astype(np.int32))\n\nclass ToxicDataset(Dataset):\n    def __init__(self, df):\n        self.df = df[[\"toxic\", \"input_word_ids\", \"input_mask\"]]\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        return {\n            'input_ids': torch.tensor(str_to_t(row[\"input_word_ids\"])),\n            'mask': torch.tensor(str_to_t(row[\"input_mask\"])),\n            'label': row[\"toxic\"]\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import time\n\nclass MyBert(nn.Module):    \n    def __init__(self):\n        super(MyBert, self).__init__()\n        self.bm = BertModel.from_pretrained(\"bert-base-multilingual-cased\")\n        self.do = nn.Dropout(p=0.2)\n        self.fc = nn.Linear(768 * 2, 1)\n\n    def forward(self, input_ids, attention_mask):\n        x = self.bm(input_ids=input_ids, attention_mask=attention_mask)[0]\n        mx, _ = torch.max(x, 1)\n        mean = torch.mean(x, 1)\n        x = torch.cat((mx, mean), 1)\n        x = self.do(x)\n        x = self.fc(x)\n        return x[:, 0]\n    \nmodel1 = MyBert()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _run():\n    device = xm.xla_device()\n    train_dataset = ToxicDataset(df_train)\n    val_dataset = ToxicDataset(df_val)\n\n    train_sampler = DistributedSampler(\n              train_dataset,\n              num_replicas=xm.xrt_world_size(),\n              rank=xm.get_ordinal(),\n        )\n\n    val_sampler = DistributedSampler(\n              val_dataset,\n              num_replicas=xm.xrt_world_size(),\n              rank=xm.get_ordinal(),\n        )\n\n    train_dataloader = DataLoader(\n            train_dataset,\n            batch_size=batch_size,\n            sampler=train_sampler,\n            num_workers=1,\n            drop_last=True\n        )\n\n    val_dataloader = DataLoader(\n            val_dataset,\n            batch_size=batch_size,\n            sampler=val_sampler,\n            num_workers=1,\n            drop_last=True\n        )\n    model = model1.to(device)\n    criterion = nn.BCEWithLogitsLoss()\n    param_optimizer = list(model.named_parameters())\n    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n    optimizer_grouped_parameters = [\n        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]\n\n    lr = 0.5e-5 * xm.xrt_world_size()\n    epochs = 2\n    num_train_steps = int(len(train_dataset) / batch_size / xm.xrt_world_size() * epochs)\n    \n    optimizer = AdamW(optimizer_grouped_parameters, lr=lr)\n    scheduler = get_linear_schedule_with_warmup(\n        optimizer,\n        num_warmup_steps=0,\n        num_training_steps=num_train_steps\n    )\n    for epoch in range(epochs):\n        para_loader = pl.ParallelLoader(train_dataloader, [device]).per_device_loader(device)\n        model.train()\n        for bn, batch in enumerate(para_loader):\n            if bn % 20 == 0:\n                xm.master_print(f\"Batch number {bn}/{len(para_loader)}\")\n            model.zero_grad()\n            input_ids = batch[\"input_ids\"].to(device).long()\n            mask = batch[\"mask\"].to(device).long()\n            labels = batch[\"label\"].to(device).float()\n            outputs = model(\n                input_ids, \n                attention_mask=mask, \n            )\n            loss = criterion(outputs, labels)\n            loss.backward()\n            xm.master_print(f'Loss on batch {bn}/{len(para_loader)}: {loss.item()}')\n            clip_grad_norm_(model.parameters(), 1.0)\n            xm.optimizer_step(optimizer)\n            if scheduler is not None:\n                scheduler.step()\n        \"\"\"model.eval()\n        para_loader = pl.ParallelLoader(val_dataloader, [device]).per_device_loader(device)\n        for bn, batch in enumerate(para_loader):\n            if bn % 200 == 0:\n                xm.master_print(f\"Batch number {bn}/{len(para_loader)}\")\n            input_ids = batch[\"input_ids\"].to(device)\n            mask = batch[\"mask\"].to(device)\n            labels = batch[\"label\"].detach().cpu().numpy()\n            with torch.no_grad():\n                logits = model(\n                    input_ids, \n                    attention_mask=mask, \n                )[0][:, 1].detach().cpu().numpy()\n            #auc = metrics.roc_auc_score(labels, logits)\n            xm.master_print(f'AUC = {auc}')\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    _run()\n\nxmp.spawn(_mp_fn, args=({},), nprocs=8, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = xm.xla_device()\nmodel = model1.eval().to(device)\nans = []\nfor idx in range(len(df_test)):\n    s = str_to_t(df_test.iloc[idx][\"input_word_ids\"])\n    m = str_to_t(df_test.iloc[idx][\"input_mask\"])\n    out = model(\n            s.long().unsqueeze(0).to(device),\n            m.long().unsqueeze(0).to(device)\n        )[0]\n    a = nn.Sigmoid()(out).item()\n    ans.append(a)\nxm.master_print(\"Fin\")\nif xm.get_ordinal() == 0:\n    with open(\"submission.csv\", \"w\") as f:\n        print(\"id,toxic\", file=f)\n        for i in range(len(ans)):\n            print(f\"{i},{ans[i]}\", file=f)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}