{"cells":[{"metadata":{},"cell_type":"markdown","source":"# XLM-RoBERTa inference\n\n## Please upvote if you found this helpful\n\nThis is just some quick an dirty inference for [this kernel](https://www.kaggle.com/tanlikesmath/xlm-roberta-pytorch-xla-tpu/). Someone will probably come out with a 8-core inference kernel."},{"metadata":{"trusted":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"16a3df23-8416-46b5-8661-c52345005b6d","_uuid":"993abe0b-2561-4abc-a6a2-b83d8dd4c846","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\n\nfrom tqdm import tqdm\nfrom collections import OrderedDict, namedtuple\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport joblib\n\nimport logging\nimport transformers\nimport sys","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nimport torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.data_parallel as dp\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\nimport warnings\nimport time\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class CustomRoberta(nn.Module):\n    def __init__(self):\n        super(CustomRoberta, self).__init__()\n        self.num_labels = 1\n        self.roberta = transformers.XLMRobertaModel.from_pretrained(\"xlm-roberta-large\", output_hidden_states=False, num_labels=1)\n        self.dropout = nn.Dropout(p=0.2)\n        self.classifier = nn.Linear(1024, self.num_labels)\n\n    def forward(self,\n                input_ids=None,\n                attention_mask=None,\n                position_ids=None,\n                head_mask=None,\n                inputs_embeds=None):\n\n        _, o2 = self.roberta(input_ids,\n                               attention_mask=attention_mask,\n                               position_ids=position_ids,\n                               head_mask=head_mask,\n                               inputs_embeds=inputs_embeds)\n        \n        #apool = torch.mean(o1, 1)\n        #mpool, _ = torch.max(o1, 1)\n        #cat = torch.cat((apool, mpool), 1)\n        #bo = self.dropout(cat)\n        logits = self.classifier(o2)       \n        outputs = logits\n        return outputs\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.XLMRobertaTokenizer.from_pretrained('xlm-roberta-large', do_lower_case=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntest = regular_encode(df.content.values, tokenizer, maxlen=192)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = CustomRoberta()\nmodel.load_state_dict(torch.load(\"../input/xlm-roberta-pytorch-xla-tpu/xlm_roberta_model.bin\"))\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = torch.utils.data.TensorDataset(torch.Tensor(test))\n\n\ntest_data_loader = torch.utils.data.DataLoader(\n    test_dataset,\n    batch_size=64,\n    drop_last=False,\n    num_workers=4,\n    shuffle=False\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = xm.xla_device()\nwith torch.no_grad():\n    fin_outputs = []\n    model.to(device)\n    for bi, d in tqdm(enumerate(test_data_loader),total=len(test_data_loader)):\n        ids = d[0]\n\n        ids = ids.to(device, dtype=torch.long)\n\n        outputs = model(\n            input_ids=ids,\n        )\n        \n        #outputs = torch.sigmoid(outputs)\n        outputs_np = outputs.cpu().detach().numpy().tolist()\n        fin_outputs.extend(outputs_np)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fin_outputs = [item for sublist in fin_outputs for item in sublist]\nsample = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nsample.loc[:, \"toxic\"] = np.array(fin_outputs)\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":4}