{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport torch\nimport pandas as pd\nfrom scipy import stats\nimport numpy as np\nimport pandas as pd\n\nfrom tqdm import tqdm\nfrom collections import OrderedDict, namedtuple\nimport torch.nn as nn\nfrom torch.optim import lr_scheduler\nimport joblib\n\nimport logging\nimport transformers\nimport sys","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"class BERTBaseUncased(nn.Module):\n    def __init__(self, bert_path):\n        super(BERTBaseUncased, self).__init__()\n        self.bert_path = bert_path\n        self.bert = transformers.BertModel.from_pretrained(self.bert_path)\n        self.bert_drop = nn.Dropout(0.3)\n        self.out = nn.Linear(768 * 2, 1)\n\n    def forward(\n            self,\n            ids,\n            mask,\n            token_type_ids\n    ):\n        o1, o2 = self.bert(\n            ids,\n            attention_mask=mask,\n            token_type_ids=token_type_ids)\n        \n        apool = torch.mean(o1, 1)\n        mpool, _ = torch.max(o1, 1)\n        cat = torch.cat((apool, mpool), 1)\n\n        bo = self.bert_drop(cat)\n        p2 = self.out(bo)\n        return p2\n\n\nclass BERTDatasetTest:\n    def __init__(self, comment_text, tokenizer, max_length):\n        self.comment_text = comment_text\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n\n    def __len__(self):\n        return len(self.comment_text)\n\n    def __getitem__(self, item):\n        comment_text = str(self.comment_text[item])\n        comment_text = \" \".join(comment_text.split())\n\n        inputs = self.tokenizer.encode_plus(\n            comment_text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_length,\n        )\n        ids = inputs[\"input_ids\"]\n        token_type_ids = inputs[\"token_type_ids\"]\n        mask = inputs[\"attention_mask\"]\n        \n        padding_length = self.max_length - len(ids)\n        \n        ids = ids + ([0] * padding_length)\n        mask = mask + ([0] * padding_length)\n        token_type_ids = token_type_ids + ([0] * padding_length)\n        \n        return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)\n        }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.BertTokenizer.from_pretrained(\"../input/bert-base-multilingual-uncased/\", do_lower_case=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = \"cuda\"\nmodel = BERTBaseUncased(bert_path=\"../input/bert-base-multilingual-uncased/\").to(device)\nmodel.load_state_dict(torch.load(\"../input/jmodelval/model.bin\"))\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_dataset = BERTDatasetTest(\n        comment_text=df.content.values,\n        tokenizer=tokenizer,\n        max_length=192\n)\n\nvalid_data_loader = torch.utils.data.DataLoader(\n    valid_dataset,\n    batch_size=64,\n    drop_last=False,\n    num_workers=4,\n    shuffle=False\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with torch.no_grad():\n    fin_outputs = []\n    for bi, d in tqdm(enumerate(valid_data_loader)):\n        ids = d[\"ids\"]\n        mask = d[\"mask\"]\n        token_type_ids = d[\"token_type_ids\"]\n\n        ids = ids.to(device, dtype=torch.long)\n        mask = mask.to(device, dtype=torch.long)\n        token_type_ids = token_type_ids.to(device, dtype=torch.long)\n\n        outputs = model(\n            ids=ids,\n            mask=mask,\n            token_type_ids=token_type_ids\n        )\n\n        outputs_np = outputs.cpu().detach().numpy().tolist()\n        fin_outputs.extend(outputs_np)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# translated english data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_en = pd.read_csv(\"../input/test-en-df/test_en.csv\")\n\nvalid_dataset = BERTDatasetTest(\n        comment_text=df_en.content_en.values,\n        tokenizer=tokenizer,\n        max_length=192\n)\n\nvalid_data_loader = torch.utils.data.DataLoader(\n    valid_dataset,\n    batch_size=64,\n    drop_last=False,\n    num_workers=4,\n    shuffle=False\n)\n\nwith torch.no_grad():\n    fin_outputs_en = []\n    for bi, d in tqdm(enumerate(valid_data_loader)):\n        ids = d[\"ids\"]\n        mask = d[\"mask\"]\n        token_type_ids = d[\"token_type_ids\"]\n\n        ids = ids.to(device, dtype=torch.long)\n        mask = mask.to(device, dtype=torch.long)\n        token_type_ids = token_type_ids.to(device, dtype=torch.long)\n\n        outputs = model(\n            ids=ids,\n            mask=mask,\n            token_type_ids=token_type_ids\n        )\n\n        outputs_np = outputs.cpu().detach().numpy().tolist()\n        fin_outputs_en.extend(outputs_np)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# another translated english data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_en2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv\")\n\nvalid_dataset = BERTDatasetTest(\n        comment_text=df_en2.translated.values,\n        tokenizer=tokenizer,\n        max_length=192\n)\n\nvalid_data_loader = torch.utils.data.DataLoader(\n    valid_dataset,\n    batch_size=64,\n    drop_last=False,\n    num_workers=4,\n    shuffle=False\n)\n\nwith torch.no_grad():\n    fin_outputs_en2 = []\n    for bi, d in tqdm(enumerate(valid_data_loader)):\n        ids = d[\"ids\"]\n        mask = d[\"mask\"]\n        token_type_ids = d[\"token_type_ids\"]\n\n        ids = ids.to(device, dtype=torch.long)\n        mask = mask.to(device, dtype=torch.long)\n        token_type_ids = token_type_ids.to(device, dtype=torch.long)\n\n        outputs = model(\n            ids=ids,\n            mask=mask,\n            token_type_ids=token_type_ids\n        )\n\n        outputs_np = outputs.cpu().detach().numpy().tolist()\n        fin_outputs_en2.extend(outputs_np)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fin_outputs_en = [item for sublist in fin_outputs_en for item in sublist]\nfin_outputs_en2 = [item for sublist in fin_outputs_en2 for item in sublist]\nfin_outputs = [item for sublist in fin_outputs for item in sublist]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nsample.loc[:, \"toxic\"] = (np.array(fin_outputs) + np.array(fin_outputs_en) + np.array(fin_outputs_en2)) / 3.0\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}