{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\n# for dirname, _, filenames in os.walk('/kaggle/input'):\n#     for filename in filenames:\n#         print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport transformers\nimport torch\nimport torch.nn as nn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class JigsawDataset:\n#     def __init__(self, comment_text, tokenizer, max_length):\n#         self.comment_text = comment_text\n#         self.tokenizer = tokenizer\n#         self.max_length = max_length\n# #         self.targets = targets\n\n#     def __len__(self):\n#         return len(self.comment_text)\n\n#     def __getitem__(self, item):\n#         comment_text = str(self.comment_text[item])\n#         comment_text = \" \".join(comment_text.split())\n\n#         inputs = self.tokenizer.encode_plus(\n#             comment_text,\n#             None,\n#             add_special_tokens=True,\n#             max_length=self.max_length,\n#         )\n#         ids = inputs[\"input_ids\"]\n#         token_type_ids = inputs[\"token_type_ids\"]\n#         mask = inputs[\"attention_mask\"]\n        \n#         padding_length = self.max_length - len(ids)\n        \n#         ids = ids + ([0] * padding_length)\n#         mask = mask + ([0] * padding_length)\n#         token_type_ids = token_type_ids + ([0] * padding_length)\n        \n#         return {\n#             'ids': torch.tensor(ids, dtype=torch.long),\n#             'mask': torch.tensor(mask, dtype=torch.long),\n#             'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long),\n#             'targets': torch.tensor(self.targets[item], dtype=torch.float)\n#         }\n    \nclass JigsawDataset:\n    def __init__(self, comment_text, tokenizer, max_length):\n        self.comment_text = comment_text\n        self.tokenizer = tokenizer\n        self.max_length = max_length\n\n    def __len__(self):\n        return len(self.comment_text)\n\n    def __getitem__(self, item):\n        comment_text = str(self.comment_text[item])\n        comment_text = \" \".join(comment_text.split())\n\n        inputs = self.tokenizer.encode_plus(\n            comment_text,\n            None,\n            add_special_tokens=True,\n            max_length=self.max_length,\n        )\n        ids = inputs[\"input_ids\"]\n        token_type_ids = inputs[\"token_type_ids\"]\n        mask = inputs[\"attention_mask\"]\n        \n        padding_length = self.max_length - len(ids)\n        \n        ids = ids + ([0] * padding_length)\n        mask = mask + ([0] * padding_length)\n        token_type_ids = token_type_ids + ([0] * padding_length)\n        \n        return {\n            'ids': torch.tensor(ids, dtype=torch.long),\n            'mask': torch.tensor(mask, dtype=torch.long),\n            'token_type_ids': torch.tensor(token_type_ids, dtype=torch.long)\n        }\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# class BERTBaseMultiUncased(nn.Module):\n#     def __init__(self, bert_path):\n#         super(BERTBaseMultiUncased, self).__init__()\n# #         super(self).__init__()\n#         self.bert_path = bert_path\n#         self.bert = transformers.BertModel.from_pretrained(self.bert_path)\n#         self.bert_drop = nn.Dropout(0.3)\n#         self.out = nn.Linear(768 * 2, 1)\n\n#     def forward(\n#             self,\n#             ids,\n#             mask,\n#             token_type_ids\n#     ):\n#         o1, o2 = self.bert(\n#             ids,\n#             attention_mask=mask,\n#             token_type_ids=token_type_ids)\n        \n#         apool = torch.mean(o1, 1)\n#         mpool, _ = torch.max(o1, 1)\n#         cat = torch.cat((apool, mpool), 1)\n\n#         bo = self.bert_drop(cat)\n#         p2 = self.out(bo)\n#         return p2\n    \nclass BERTBaseMultiUncased(nn.Module):\n    def __init__(self, bert_path):\n        super(BERTBaseMultiUncased, self).__init__()\n        self.bert_path = bert_path\n        self.bert = transformers.BertModel.from_pretrained(self.bert_path)\n        self.bert_drop = nn.Dropout(0.3)\n        self.out = nn.Linear(768 * 2, 1)\n\n    def forward(\n            self,\n            ids,\n            mask,\n            token_type_ids\n    ):\n        o1, o2 = self.bert(\n            ids,\n            attention_mask=mask,\n            token_type_ids=token_type_ids)\n        \n        apool = torch.mean(o1, 1)\n        mpool, _ = torch.max(o1, 1)\n        cat = torch.cat((apool, mpool), 1)\n\n        bo = self.bert_drop(cat)\n        p2 = self.out(bo)\n        return p2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.BertTokenizer.from_pretrained(\"../input/bert-base-multilingual-uncased/\", do_lower_case=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"os.listdir()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = \"cuda\"\nmodel = BERTBaseMultiUncased(bert_path=\"../input/bert-base-multilingual-uncased/\").to(device)\nmodel.load_state_dict(torch.load(\"../input/jigsawmulti-bertmulti/model_multi.bin\"))\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_dataset = JigsawDataset(\n        comment_text=df.content.values,\n        tokenizer=tokenizer,\n        max_length=192\n)\n\nvalid_data_loader = torch.utils.data.DataLoader(\n    valid_dataset,\n    batch_size=64,\n    drop_last=False,\n    num_workers=1,\n    shuffle=False\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with torch.no_grad():\n    fin_outputs = []\n    for bi, d in tqdm(enumerate(valid_data_loader)):\n        ids = d[\"ids\"]\n        mask = d[\"mask\"]\n        token_type_ids = d[\"token_type_ids\"]\n\n        ids = ids.to(device, dtype=torch.long)\n        mask = mask.to(device, dtype=torch.long)\n        token_type_ids = token_type_ids.to(device, dtype=torch.long)\n\n        outputs = model(\n            ids=ids,\n            mask=mask,\n            token_type_ids=token_type_ids\n        )\n\n        outputs_np = outputs.cpu().detach().numpy().tolist()\n        fin_outputs.extend(outputs_np)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nsample.loc[:, \"toxic\"] = fin_outputs\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"DONE\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}