{"cells":[{"metadata":{"trusted":true,"_kg_hide-output":true},"cell_type":"code","source":"!curl https://raw.githubusercontent.com/pytorch/xla/master/contrib/scripts/env-setup.py -o pytorch-xla-env-setup.py\n!python pytorch-xla-env-setup.py --version nightly --apt-packages libomp5 libopenblas-dev","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"993abe0b-2561-4abc-a6a2-b83d8dd4c846","_cell_guid":"16a3df23-8416-46b5-8661-c52345005b6d","trusted":true},"cell_type":"code","source":"import os\nos.environ['XLA_USE_BF16']=\"1\"\nos.environ['XLA_TENSOR_ALLOCATOR_MAXSIZE'] = '100000000'\n\nimport torch\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\n\nfrom tqdm import tqdm\nfrom collections import OrderedDict, namedtuple\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport joblib\n\nimport logging\nimport transformers\nfrom transformers import AdamW, get_linear_schedule_with_warmup, get_constant_schedule\nfrom transformers import AutoTokenizer, AutoModel, AutoConfig\nimport sys\nfrom sklearn import metrics, model_selection\nfrom fastai.text import *","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"993abe0b-2561-4abc-a6a2-b83d8dd4c846","_cell_guid":"16a3df23-8416-46b5-8661-c52345005b6d","trusted":true},"cell_type":"code","source":"import warnings\nimport torch_xla\nimport torch_xla.debug.metrics as met\nimport torch_xla.distributed.parallel_loader as pl\nimport torch_xla.utils.utils as xu\nimport torch_xla.core.xla_model as xm\nimport torch_xla.distributed.xla_multiprocessing as xmp\nimport torch_xla.test.test_utils as test_utils\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"class JigsawArrayDataset(torch.utils.data.Dataset):\n    def __init__(self, input_ids:np.array, attention_mask:np.array, toxic:np.array=None, text_id=None):\n        self.input_ids = input_ids\n        self.attention_mask = attention_mask\n        self.toxic = toxic\n        self.text_id = text_id\n    \n    def __getitem__(self, idx):\n        xb = (tensor(self.input_ids[idx]), tensor(self.attention_mask[idx]))\n        yb = tensor(0.) if self.toxic is None else tensor(self.toxic[idx])\n        yb = yb if self.text_id is None else (yb, self.text_id[idx])\n        return xb,yb    \n        \n    def __len__(self):\n        return len(self.input_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"XLM_PROCESSED_PATH = Path(\"/kaggle/input/xlmrobertabase/xlm_roberta_processed/\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"XLM_PROCESSED_PATH.ls()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_xlm_roberta_base():\n    conf = AutoConfig.from_pretrained(\"xlm-roberta-base\")\n    conf.output_hidden_states = True\n    model = AutoModel.from_config(config=conf)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Head(Module):\n    \"Concat Pool over sequence\"\n    def __init__(self, p=0.5):\n        self.d0 = nn.Dropout(p)\n        self.l0 = nn.Linear(1536*2, 2)\n        \n    def forward(self, x):\n        x = self.d0(x)\n        x = torch.cat([x.permute(0,-1,-2).mean(-1), \n                       x.permute(0,-1,-2).max(-1).values], -1)\n        x = self.l0(x) \n        return x\n\nclass JigsawModel(Module):\n    def __init__(self, model, head):\n        self.sequence_model = model\n        self.head = head\n\n    def forward(self, *xargs):\n        inp = {}\n        inp[\"input_ids\"] = xargs[0]\n        inp[\"attention_mask\"] = xargs[1]\n        _, _, hidden_states = self.sequence_model(**inp)\n        # feed last 2 hidden states\n        x = torch.cat(hidden_states[-2:], -1)\n        return self.head(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = get_xlm_roberta_base()\nhead = Head()\njigsaw_model = JigsawModel(model, head)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### load model"},{"metadata":{"trusted":true},"cell_type":"code","source":"state_dict = torch.load(\"/kaggle/input/xlmrobertatoxicengmodel/model_finetuned-translated-data.bin\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jigsaw_model.load_state_dict(state_dict)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prediction"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score\n\ndef predict_fn(data_loader, model, device, num_batches):\n    model.eval()\n    preds, text_ids = [], []\n   \n    with torch.no_grad():\n#         tk0 = tqdm(data_loader, total=num_batches, desc=\"Predicting\", disable=not xm.is_master_ordinal())\n        tk0 = tqdm(data_loader, total=num_batches, desc=\"Predicting\", disable=not xm.is_master_ordinal())\n        for bi, (xb,yb) in enumerate(tk0):\n\n            input_ids, attention_mask = xb\n            input_ids = input_ids.to(device, dtype=torch.long)\n            attention_mask = attention_mask.to(device, dtype=torch.long)\n            out = model(input_ids, attention_mask)\n            \n            preds.append(to_cpu(out.softmax(-1)[:,1]))\n            text_ids.append(to_cpu(yb[1]))\n\n    return preds, text_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from torch.utils.data import DataLoader","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(test_ds):\n    device = xm.xla_device()\n    model = jigsaw_model.to(device)\n        \n\n    test_dl = torch.utils.data.DataLoader(\n        test_ds,\n        batch_size=128,\n        shuffle=False,\n        num_workers=4\n    )\n    \n    preds, text_ids = predict_fn(test_dl, model, device, len(test_dl))\n    return preds, text_ids","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ds = JigsawArrayDataset(\n    input_ids = np.load(XLM_PROCESSED_PATH/'test_inputs/input_ids.npy'),\n    attention_mask = np.load(XLM_PROCESSED_PATH/'test_inputs/attention_mask.npy'),\n    text_id = test_df['id'].values\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1, text_ids = run(test_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_toxic = np.load(tokenized_path+'df_train_toxic.npy',mmap_mode='r')\ntest_ds = JigsawArrayDataset(\n    input_ids = np.load(XLM_PROCESSED_PATH/'translated_test_inputs/input_ids.npy'),\n    attention_mask = np.load(XLM_PROCESSED_PATH/'translated_test_inputs/attention_mask.npy'),\n    text_id = test_df['id'].values\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds2, text_ids = run(test_ds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1 = to_np(torch.cat(preds1).view(-1))\npreds2= to_np(torch.cat(preds2).view(-1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.corrcoef(preds1, preds2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = (preds1 + preds2) / 2\ntext_ids = to_np(torch.cat(text_ids).view(-1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.hist(preds)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submit"},{"metadata":{"trusted":true},"cell_type":"code","source":"subdf = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nsubdf['toxic'] = preds\nsubdf.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### fin"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}