{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom pathlib import Path\nfrom sklearn.utils import shuffle\nfrom sklearn import metrics\n# imports pytorch\nimport torch\nimport torch.nn as nn\nfrom torch.utils.data.distributed import DistributedSampler\nfrom torch.utils.data import DataLoader\n# # imports the torch_xla package\n# import torch_xla\n# import torch_xla.distributed.parallel_loader as pl\n# import torch_xla.core.xla_model as xm\n# import torch_xla.distributed.xla_multiprocessing as xmp\n\nfrom tqdm import tqdm\n\n#transformers\nimport transformers\nfrom transformers import AdamW, get_linear_schedule_with_warmup, get_constant_schedule\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BERT_ROOT = \"../input/bertbasemultilingualuncased/\"\nMAX_LEN = 192","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_ROOT = Path(\"..\")/\"input\"/ \"jigsaw-multilingual-toxic-comment-classification/\"\n\ntest,sample = [pd.read_csv(DATA_ROOT / fname) for fname in [\"test.csv\",\"sample_submission.csv\"]]\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DatasetClass:\n    def __init__(self, text,tokenizer, max_length):\n        self.text = text\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n        \n\n    def __len__(self):\n        return len(self.text)\n\n    def __getitem__(self, item):\n        text = str(self.text[item])\n        text = \" \".join(text.split())\n\n        inputs = self.tokenizer.encode_plus(\n            text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_length,\n        )\n        ids = inputs[\"input_ids\"]\n        token_type_ids = inputs[\"token_type_ids\"]\n        mask = inputs[\"attention_mask\"]\n\n        padding_length = self.max_length - len(ids)\n\n        ids = ids + ([0] * padding_length)\n        mask = mask + ([0] * padding_length)\n        token_type_ids = token_type_ids + ([0] * padding_length)\n\n        return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)\n            \n        }\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nclass Bert_fn(nn.Module):\n    def __init__(self, bert_path):\n        super(Bert_fn, self).__init__()\n        self.bert_path = bert_path\n        self.bert = transformers.BertModel.from_pretrained(self.bert_path)\n        self.bert_drop = nn.Dropout(0.3)\n        self.out = nn.Linear(768 * 2, 1)\n\n    def forward(\n            self,\n            ids,\n            mask,\n            token_type_ids\n    ):\n        o1, o2 = self.bert(\n            ids,\n            attention_mask=mask,\n            token_type_ids=token_type_ids)\n\n        apool = torch.mean(o1, 1)\n        mpool, _ = torch.max(o1, 1)\n        cat = torch.cat((apool, mpool), 1)\n\n        bo = self.bert_drop(cat)\n        p2 = self.out(bo)\n        return p2\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BERT_MODEL = Bert_fn(bert_path=\"../input/bertbasemultilingualuncased/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.BertTokenizer.from_pretrained(BERT_ROOT, do_lower_case=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = \"cuda\"\nmodel = BERT_MODEL.to(device)\nmodel.load_state_dict(torch.load(\"../input/pytorch-multilingual-tpu/model.bin\"))\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = DatasetClass(\n        text=test.content.values,\n        tokenizer=tokenizer,\n        max_length=MAX_LEN\n    )\n\ntest_loader = torch.utils.data.DataLoader(\n        test_dataset,\n        batch_size=64,\n        shuffle = False,\n        drop_last=False,\n        num_workers=4\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with torch.no_grad():\n    final_outputs = []\n    for bi, d in tqdm(enumerate(test_loader)):\n        ids = d[\"ids\"]\n        mask = d[\"mask\"]\n        token_type_ids = d[\"token_type_ids\"]\n\n        ids = ids.to(device, dtype=torch.long)\n        mask = mask.to(device, dtype=torch.long)\n        token_type_ids = token_type_ids.to(device, dtype=torch.long)\n\n        outputs = model(\n            ids=ids,\n            mask=mask,\n            token_type_ids=token_type_ids\n        )\n\n        outputs_np = outputs.cpu().detach().numpy().tolist()\n        final_outputs.extend(outputs_np)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Translated Test Data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"test1 = pd.read_csv(\"../input/translated-test-data/test_en.csv\")\ntest1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test1_dataset = DatasetClass(\n        text=test1.content_en.values,\n        tokenizer=tokenizer,\n        max_length=MAX_LEN\n    )\n\ntest1_loader = torch.utils.data.DataLoader(\n        test1_dataset,\n        batch_size=64,\n        shuffle = False,\n        drop_last=False,\n        num_workers=4\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with torch.no_grad():\n    final_outputs1 = []\n    for bi, d in tqdm(enumerate(test1_loader)):\n        ids = d[\"ids\"]\n        mask = d[\"mask\"]\n        token_type_ids = d[\"token_type_ids\"]\n\n        ids = ids.to(device, dtype=torch.long)\n        mask = mask.to(device, dtype=torch.long)\n        token_type_ids = token_type_ids.to(device, dtype=torch.long)\n\n        outputs = model(\n            ids=ids,\n            mask=mask,\n            token_type_ids=token_type_ids\n        )\n\n        outputs_np = outputs.cpu().detach().numpy().tolist()\n        final_outputs1.extend(outputs_np)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.loc[:, \"toxic\"] = (np.array(final_outputs)+ np.array(final_outputs)) / 2.0\nsample.to_csv(\"submission.csv\", index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}