{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os.path\nimport os\nimport shutil\nimport sys\nimport time\nfrom datetime import datetime\nimport glob\n\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom sklearn import metrics\nimport torch\nimport torch.nn as nn\nfrom torch.nn.utils.clip_grad import clip_grad_norm_\nfrom torch.optim import Adam\nfrom torch.utils.data import Dataset, DataLoader\nfrom torch.utils.data.distributed import DistributedSampler\nfrom transformers import *","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"home_dir = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\"\ndf = pd.read_csv(os.path.join(home_dir, \"jigsaw-toxic-comment-train-processed-seqlen128.csv\"))\ndf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train = df\ndf_val = pd.read_csv(os.path.join(home_dir, \"validation-processed-seqlen128.csv\"))\ndf_test = pd.read_csv(os.path.join(home_dir, \"test-processed-seqlen128.csv\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[\"toxic\"].hist()\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def str_to_t(s):\n    return torch.tensor(np.array(s[1:-1].split(',')).astype(np.int32))\n\nclass ToxicDataset(Dataset):\n    def __init__(self, df):\n        self.df = df\n        \n    def __len__(self):\n        return len(self.df)\n    \n    def __getitem__(self, idx):\n        row = self.df.iloc[idx]\n        try:\n            label = row['toxic']\n        except Exception:\n            label = -1 #test dataset\n        return {\n            'id': idx,\n            'input_ids': torch.tensor(str_to_t(row[\"input_word_ids\"])),\n            'mask': torch.tensor(str_to_t(row[\"input_mask\"])),\n            'label': label\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MyBert(nn.Module):    \n    def __init__(self):\n        super(MyBert, self).__init__()\n        self.bm = BertModel.from_pretrained(\"bert-base-multilingual-cased\")\n        self.do = nn.Dropout(p=0.2)\n        self.fc = nn.Linear(768 * 2, 1)\n\n    def forward(self, input_ids, attention_mask):\n        x = self.bm(input_ids=input_ids, attention_mask=attention_mask)[0]\n        mx, _ = torch.max(x, 1)\n        mean = torch.mean(x, 1)\n        x = torch.cat((mx, mean), 1)\n        x = self.do(x)\n        x = self.fc(x)\n        return x[:, 0]\n    \nmodel = MyBert()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clear_dir(d):\n    folder = d\n    for filename in os.listdir(folder):\n        file_path = os.path.join(folder, filename)\n        try:\n            if os.path.isfile(file_path) or os.path.islink(file_path):\n                os.unlink(file_path)\n            elif os.path.isdir(file_path):\n                shutil.rmtree(file_path)\n        except Exception as e:\n            print('Failed to delete %s. Reason: %s' % (file_path, e))\n\ntry:\n    os.mkdir('./tmp')\nexcept Exception:\n    clear_dir('./tmp')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _run():\n    device = xm.xla_device()\n    train_dataset = ToxicDataset(df_train)\n    val_dataset = ToxicDataset(df_val)\n\n    train_sampler = DistributedSampler(\n              train_dataset,\n              num_replicas=xm.xrt_world_size(),\n              rank=xm.get_ordinal(),\n        )\n\n    val_sampler = DistributedSampler(\n              val_dataset,\n              num_replicas=xm.xrt_world_size(),\n              rank=xm.get_ordinal(),\n        )\n\n    train_dataloader = DataLoader(\n            train_dataset,\n            batch_size=batch_size,\n            sampler=train_sampler,\n            num_workers=1,\n            drop_last=True\n        )\n\n    val_dataloader = DataLoader(\n            val_dataset,\n            batch_size=batch_size,\n            sampler=val_sampler,\n            num_workers=1,\n            drop_last=True\n        )\n    model.to(device)\n    criterion = nn.BCEWithLogitsLoss()\n    param_optimizer = list(model.named_parameters())\n    no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n    optimizer_grouped_parameters = [\n        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.001},\n        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}]\n\n    lr = 0.5e-5 * xm.xrt_world_size()\n    epochs = 2\n    num_train_steps = int(len(train_dataset) / batch_size / xm.xrt_world_size() * epochs)\n    \n    optimizer = AdamW(optimizer_grouped_parameters, lr=lr)\n    scheduler = get_linear_schedule_with_warmup(\n        optimizer,\n        num_warmup_steps=0,\n        num_training_steps=num_train_steps\n    )\n    for epoch in range(epochs):\n        para_loader = pl.ParallelLoader(train_dataloader, [device]).per_device_loader(device)\n        model.train()\n        for bn, batch in enumerate(para_loader):\n            if bn % 20 == 0:\n                xm.master_print(f\"Batch number {bn}/{len(para_loader)}\")\n            model.zero_grad()\n            input_ids = batch[\"input_ids\"].to(device).long()\n            mask = batch[\"mask\"].to(device).long()\n            labels = batch[\"label\"].to(device).float()\n            outputs = model(\n                input_ids, \n                attention_mask=mask, \n            )\n            loss = criterion(outputs, labels)\n            loss.backward()\n            xm.master_print(f'Loss on batch {bn}/{len(para_loader)}: {loss.item()}')\n            clip_grad_norm_(model.parameters(), 1.0)\n            xm.optimizer_step(optimizer)\n            if scheduler is not None:\n                scheduler.step()\n    xm.save(model.state_dict(), f\"model.pt\")\n    xm.master_print(\"kek\")\n    '''model.eval()\n    para_loader = pl.ParallelLoader(val_dataloader, [device]).per_device_loader(device)\n    for bn, batch in enumerate(para_loader):\n        if bn % 200 == 0:\n            xm.master_print(f\"Batch number {bn}/{len(para_loader)}\")\n        input_ids = batch[\"input_ids\"].to(device)\n        mask = batch[\"mask\"].to(device)\n        labels = batch[\"label\"].detach().cpu().numpy()\n        with torch.no_grad():\n            logits = model(\n                input_ids, \n                attention_mask=mask, \n            )[0][:, 1].detach().cpu().numpy()\n        #auc = metrics.roc_auc_score(labels, logits)\n        xm.master_print(f'AUC = {auc}')\n    test_dataset = ToxicDataset(df_test)\n    test_sampler = DistributedSampler(\n          test_dataset,\n          num_replicas=xm.xrt_world_size(),\n          rank=xm.get_ordinal(),\n    )\ntest_dataloader = DataLoader(\n        test_dataset,\n        batch_size=batch_size,\n        sampler=test_sampler,\n        num_workers=1,\n        drop_last=True\n    )\nmodel.eval()\npara_loader = pl.ParallelLoader(test_dataloader, [device]).per_device_loader(device)\nxm.master_print(\"gonna print\")\nfor bn, batch in enumerate(para_loader):\n    input_ids = batch[\"input_ids\"].to(device).long()\n    mask = batch[\"mask\"].to(device).long()\n    answers = model(\n            input_ids,\n            attention_mask=mask\n        )\n    ids = batch[\"id\"]\n    ans_df = pd.Dataframe({'id': ids.numpy(), 'toxic': answers.numpy()})\n    ans_df.to_csv(f\"./tmp/sub_{datetime.utcnow().microsecond}.csv\")\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def _mp_fn(rank, flags):\n    torch.set_default_tensor_type('torch.FloatTensor')\n    _run()\n\nxmp.spawn(_mp_fn, args=({},), nprocs=8, start_method='fork')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del model\nmodel = MyBert()\nmodel.load_state_dict(torch.load('model.pt'))\ndevice = xm.xla_device()\nmodel.eval().to(device)\nans = []\ntest_dataset = ToxicDataset(df_test)\ntest_dataloader = DataLoader(\n            test_dataset,\n            batch_size=batch_size\n        )\n\nfor batch in test_dataloader:\n    input_ids = batch[\"input_ids\"].to(device).long()\n    mask = batch[\"mask\"].to(device).long()\n    outputs = nn.Sigmoid()(model(\n        input_ids, \n        attention_mask=mask, \n    ))\n    for i in range(outputs.size()[0]):\n        ans.append(outputs[i].item())\nxm.master_print(\"Fin\")\nans_df = pd.DataFrame({\"id\": list(range(len(ans))), \"toxic\": ans})\nans_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}