{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from __future__ import absolute_import,division,print_function\nimport torch.utils.data\nfrom tqdm import tqdm\nimport warnings\nwarnings.filterwarnings(action='once')\ndevice = torch.device('cuda')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import sys\npackage_dir = \"../input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT\"\nsys.path.append(package_dir)\n\nfrom pytorch_pretrained_bert import BertTokenizer,BertForSequenceClassification,BertAdam,BertConfig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_lines(example,max_seq_length,tokenizer):\n    max_seq_length -= 2\n    all_tokens = []\n    longer = 0\n    for text in tqdm(example):\n        tokens_a = tokenizer.tokenize(text)\n        if len(tokens_a)>max_seq_length:\n            tokens_a = tokens_a[:max_seq_length]\n            longer +=1\n        one_token = tokenizer.convert_tokens_to_ids([\"[CLS]\"] + tokens_a + [\"[SEP]\"] \n                                                   )+[0] * (max_seq_length - len(tokens_a))\n        all_tokens.append(one_token)\n        \n    return np.array(all_tokens)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_seq_len = 220\nseed = 1234\nbatch_size = 32\nbert_model_path = '../input/bert-pytorch-from-yuval/'\nBERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(seed)\ntorch.manual_seed(seed)\ntorch.cuda.manual_seed(seed)\ntorch.backends.cudnn.deterministic = True\n\nbert_config = BertConfig(bert_model_path+'bert_config.json')\ntokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[\"content\"].fillna(\"DUMMY_VALUE\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")\ntest_df['content'] = test_df['content'].astype(str) \nX_test = convert_lines(test_df[\"content\"].fillna(\"DUMMY_VALUE\"), max_seq_len, tokenizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = BertForSequenceClassification(bert_config,num_labels=1)\nmodel.load_state_dict(torch.load(\"../input/bert-pytorch-from-yuval/bert_pytorch.bin\"))\nmodel.to(device)\nfor param in model.parameters():\n    param.requires_grad = False\nmodel.eval()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_preds = np.zeros((len(X_test)))\ntest = torch.utils.data.TensorDataset(torch.tensor(X_test, dtype=torch.long))\ntest_loader = torch.utils.data.DataLoader(test, batch_size=32, shuffle=False)\ntk0 = tqdm(test_loader)\nfor i, (x_batch,) in enumerate(tk0):\n    pred = model(x_batch.to(device), attention_mask=(x_batch > 0).to(device), labels=None)\n    test_preds[i * 32:(i + 1) * 32] = pred[:, 0].detach().cpu().squeeze().numpy()\n\ntest_pred = torch.sigmoid(torch.tensor(test_preds)).numpy().ravel()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = pd.DataFrame.from_dict({\n    'id': test_df['id'],\n    'toxic': test_pred\n})\nsubmission.to_csv('/kaggle/working/submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}