{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from __future__ import absolute_import, division\n\nimport os\nimport time\nimport numpy as np\nimport pandas as pd\nimport gensim\nfrom tqdm import tqdm\nfrom nltk.stem import PorterStemmer\nps = PorterStemmer()\nfrom nltk.stem.lancaster import LancasterStemmer\nlc = LancasterStemmer()\nfrom nltk.stem import SnowballStemmer\nsb = SnowballStemmer(\"english\")\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Loading data ...\")\ntrain = pd.read_csv('../input/quora-insincere-questions-classification/train.csv').fillna(' ')\ntest = pd.read_csv('../input/quora-insincere-questions-classification/test.csv').fillna(' ')\ntrain_texts = train['question_text']\ntest_texts = test['question_text']\ntext_list = pd.concat([train_texts, test_texts])\ntrain_labels=train['question_text']\ny = train['target'].values\nnum_train_data = y.shape[0]\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pytorch-pretrained-bert","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# BERT imports\nimport torch \nimport torch.nn as nn\nfrom torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.model_selection import train_test_split\nfrom pytorch_pretrained_bert import BertTokenizer, BertConfig\nfrom pytorch_pretrained_bert import BertAdam,BertModel, BertForSequenceClassification\nfrom tqdm import tqdm, trange\nimport pandas as pd\nimport io\nimport numpy as np\nimport matplotlib.pyplot as plt\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\ntrain_tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t)[:511], train_texts))\ntest_tokens = list(map(lambda t: ['[CLS]'] + tokenizer.tokenize(t)[:511], test_texts))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tokens_ids = list(map(tokenizer.convert_tokens_to_ids, train_tokens))\ntest_tokens_ids = list(map(tokenizer.convert_tokens_to_ids, test_tokens))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_tokens_ids = pad_sequences(train_tokens_ids, maxlen=128, truncating=\"post\", padding=\"post\", dtype=\"int\")\ntest_tokens_ids = pad_sequences(test_tokens_ids, maxlen=128, truncating=\"post\", padding=\"post\", dtype=\"int\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_y = np.array(y) == 'pos'\ntrain_y=y.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_y=np.zeros(test.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class BertBinaryClassifier(nn.Module):\n    def __init__(self, dropout=0.1):\n        super(BertBinaryClassifier, self).__init__()\n        self.bert = BertModel.from_pretrained('bert-base-uncased')\n        self.linear = nn.Linear(768, 1)\n        self.sigmoid = nn.Sigmoid()\n        #self.logsoftmax = nn.LogSoftmax(dim=1)\n        #self.softmax = nn.Softmax(dim=1)\n    \n    def forward(self, tokens):\n        _, pooled_output = self.bert(tokens, output_all_encoded_layers=False)\n        linear_output = self.linear(pooled_output)\n        proba = self.sigmoid(linear_output)\n        #proba=self.logsoftmax(linear_output)\n        #proba = self.softmax(linear_output)\n        return proba\n        #return linear_output\n        \n    def freeze_bert_encoder(self):\n        for param in self.bert.parameters():\n            param.requires_grad = False\n    \n    def unfreeze_bert_encoder(self):\n        for param in self.bert.parameters():\n            param.requires_grad = True","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"BATCH_SIZE=32\ntest_tokens_tensor = torch.tensor(test_tokens_ids[:1000])\ntest_y_tensor = torch.tensor(test_y[:1000].reshape(-1, 1)).float()\ntest_dataset = TensorDataset(test_tokens_tensor, test_y_tensor)\ntest_sampler = SequentialSampler(test_dataset)\ntest_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=BATCH_SIZE)"},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE=32\ntrain_tokens_tensor = torch.tensor(train_tokens_ids)\ntrain_y_tensor = torch.tensor(train_y.reshape(-1, 1)).float()\ntest_tokens_tensor = torch.tensor(test_tokens_ids)\ntest_y_tensor = torch.tensor(test_y.reshape(-1, 1)).float()\ntrain_dataset = TensorDataset(train_tokens_tensor, train_y_tensor)\ntrain_sampler = RandomSampler(train_dataset)\ntrain_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=BATCH_SIZE)\ntest_dataset = TensorDataset(test_tokens_tensor, test_y_tensor)\ntest_sampler = SequentialSampler(test_dataset)\ntest_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\")\nn_gpu = torch.cuda.device_count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_gpu","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time=time.time()\nEPOCHS=4\nbert_clf = BertBinaryClassifier()\nbert_clf.to(device)\nbert_clf.freeze_bert_encoder()\nbert_clf = bert_clf.cuda()\noptimizer = BertAdam(bert_clf.parameters(), lr=3e-6) \nbert_clf.train()\n\nfor epoch_num in range(EPOCHS):\n    for step_num, batch_data in enumerate(train_dataloader):\n        token_ids, labels = tuple(t.to(device) for t   in batch_data)\n        probas = bert_clf(token_ids)\n        #_, probas = torch.max(probas, 1)\n        loss_func = nn.BCELoss()\n        \n        batch_loss = loss_func(probas, labels)\n        bert_clf.freeze_bert_encoder()\n        bert_clf.zero_grad()\n        batch_loss.backward() \n        optimizer.step()\n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"torch.save({\n            'epoch': epoch_num,\n            'model_state_dict': bert_clf.state_dict(),\n            'optimizer_state_dict': optimizer.state_dict(),\n            'loss': batch_loss,\n            }, \"Quora_Bert_Pytorch.pth\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"model = BertBinaryClassifier()\noptimizer = BertAdam(model.parameters(), lr=3e-6)\n\ncheckpoint = torch.load(\"/kaggle/input/model-quora-bert/Quora_Bert_Pytorch.pth\")\nmodel.load_state_dict(checkpoint['model_state_dict'],strict=False)\noptimizer.load_state_dict(checkpoint['optimizer_state_dict'])\nepoch = checkpoint['epoch']\nloss = checkpoint['loss']\n\nmodel.eval()\n# - or -\n#model.train()"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"bert_clf = BertBinaryClassifier()\nbert_clf.load_state_dict(torch.load(\"/kaggle/input/model-quora-bert/Quora_Bert_Pytorch.pth\")['model_state_dict'],strict=False)\nbert_clf.to(device)\nbert_clf.eval()"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nbert_clf.eval()\nres = []\nfor step_num, batch_data in enumerate(test_dataloader):\n    token_ids,labels = tuple(t.to(device) for t in batch_data)\n    with torch.no_grad():\n        logits = bert_clf(token_ids)\n    logits = logits.detach().cpu().numpy()\n    for t in logits:\n        if t>0.5:\n            res.append(1)\n        else:\n            res.append(0)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#　For Submission\n\ntest['prediction'] = res\n\n\nsubmission = test[[\"qid\",\"prediction\"]]\n\n#submission.columns = ['Id', 'Category']\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}