{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install transformers","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom transformers import BertTokenizer, BertModel, BertForMaskedLM\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nimport torch\nfrom transformers import *\nfrom transformers import BertTokenizer, BertModel,BertForSequenceClassification,AdamW\nfrom torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\nfrom keras.preprocessing.sequence import pad_sequences\nfrom sklearn.model_selection import train_test_split\nfrom tqdm import tqdm, trange\nimport tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device_name = tf.test.gpu_device_name()\nif device_name != '/device:GPU:0':\n  raise SystemError('GPU device not found')\nprint('Found GPU at: {}'.format(device_name))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\nn_gpu = torch.cuda.device_count()\ntorch.cuda.get_device_name(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set the maximum sequence length.\nMAX_LEN = 128","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"sample_submission = pd.read_csv(\"../input/quora-insincere-questions-classification/sample_submission.csv\")\nall_test = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\nall_train = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_train =  all_train.loc[:10000,:]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#creates list of sentences and labels\nsentences = sample_train.question_text.values\nlabels = sample_train.target.values\n\n#initialize BERT tokenizer\ntokenizer = BertTokenizer.from_pretrained('bert-base-uncased',do_lower_case=True)\n\n#user tokenizer to convert sentences into tokenizer\ninput_ids  = [tokenizer.encode(sent,add_special_tokens=True,max_length=MAX_LEN) for sent in sentences]\n\n# Pad our input tokens\ninput_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n\n# Create attention masks\nattention_masks = []\n\n# Create a mask of 1s for each token followed by 0s for padding\nfor seq in input_ids:\n  seq_mask = [float(i>0) for i in seq]\n  attention_masks.append(seq_mask)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train and Validation split"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use train_test_split to split our data into train and validation sets for training\n\ntrain_inputs, validation_inputs, train_labels, validation_labels = train_test_split(input_ids, labels, \n                                                            random_state=2018, test_size=0.1)\ntrain_masks, validation_masks, _, _ = train_test_split(attention_masks, input_ids,\n                                             random_state=2018, test_size=0.1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Convert into Tensors"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert all of our data into torch tensors, the required datatype for our model\n\ntrain_inputs = torch.tensor(train_inputs)\nvalidation_inputs = torch.tensor(validation_inputs)\ntrain_labels = torch.tensor(train_labels)\nvalidation_labels = torch.tensor(validation_labels)\ntrain_masks = torch.tensor(train_masks)\nvalidation_masks = torch.tensor(validation_masks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Select a batch size for training. For fine-tuning BERT on a specific task, the authors recommend a batch size of 16 or 32\nbatch_size = 16\n\n# Create an iterator of our data with torch DataLoader. This helps save on memory during training because, unlike a for loop, \n# with an iterator the entire dataset does not need to be loaded into memory\n\ntrain_data = TensorDataset(train_inputs, train_masks, train_labels)\ntrain_sampler = RandomSampler(train_data)\ntrain_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n\nvalidation_data = TensorDataset(validation_inputs, validation_masks, validation_labels)\nvalidation_sampler = SequentialSampler(validation_data)\nvalidation_dataloader = DataLoader(validation_data, sampler=validation_sampler, batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer on top. \n\nmodel = BertForSequenceClassification.from_pretrained(\"bert-base-uncased\", num_labels=2)\nmodel.cuda()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"param_optimizer = list(model.named_parameters())\nno_decay = ['bias', 'gamma', 'beta']\noptimizer_grouped_parameters = [\n    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],\n     'weight_decay_rate': 0.01},\n    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],\n     'weight_decay_rate': 0.0}\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"optimizer = AdamW(optimizer_grouped_parameters, lr=2e-5, eps=10e-8)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Function to calculate the accuracy of our predictions vs labels\ndef flat_accuracy(preds, labels):\n    pred_flat = np.argmax(preds, axis=1).flatten()\n    labels_flat = labels.flatten()\n    return np.sum(pred_flat == labels_flat) / len(labels_flat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# # Store our loss and accuracy for plotting\ntrain_loss_set = []\n\n# # Number of training epochs (authors recommend between 2 and 4)\nepochs = 5\n\n# trange is a tqdm wrapper around the normal python range\nfor _ in trange(epochs, desc=\"Epoch\"):\n  \n  \n  # Training\n  \n  # Set our model to training mode (as opposed to evaluation mode)\n  model.train()\n  \n  # Tracking variables\n  tr_loss = 0\n  nb_tr_examples, nb_tr_steps = 0, 0\n  \n  # Train the data for one epoch\n  for step, batch in enumerate(train_dataloader):\n    # Add batch to GPU\n    batch = tuple(t.to(device) for t in batch)\n    # Unpack the inputs from our dataloader\n    b_input_ids, b_input_mask, b_labels = batch\n    # Clear out the gradients (by default they accumulate)\n    optimizer.zero_grad()\n    # Forward pass\n    # loss = model(b_input_ids, token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n    outputs = model(b_input_ids,token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n    loss = outputs[0]\n    train_loss_set.append(loss.item())    \n    # Backward pass\n    loss.backward()\n    # Update parameters and take a step using the computed gradient\n    optimizer.step()\n    \n    \n    # Update tracking variables\n    tr_loss += loss.item()\n    nb_tr_examples += b_input_ids.size(0)\n    nb_tr_steps += 1\n\n  print(\"Train loss: {}\".format(tr_loss/nb_tr_steps))\n    \n    \n  # Validation\n\n  # Put model in evaluation mode to evaluate loss on the validation set\n  model.eval()\n\n  # Tracking variables \n  eval_loss, eval_accuracy = 0, 0\n  nb_eval_steps, nb_eval_examples = 0, 0\n\n  # Evaluate data for one epoch\n  for batch in validation_dataloader:\n    # Add batch to GPU\n    batch = tuple(t.to(device) for t in batch)\n    # Unpack the inputs from our dataloader\n    b_input_ids, b_input_mask, b_labels = batch\n    # Telling the model not to compute or store gradients, saving memory and speeding up validation\n    with torch.no_grad():\n      # Forward pass, calculate logit predictions\n      outputs =  model(b_input_ids,token_type_ids=None, attention_mask=b_input_mask, labels=b_labels)\n      loss, logits = outputs[:2]\n    \n    # Move logits and labels to CPU\n    logits = logits.detach().cpu().numpy()\n    label_ids = b_labels.to('cpu').numpy()\n\n    tmp_eval_accuracy = flat_accuracy(logits, label_ids)\n    \n    eval_accuracy += tmp_eval_accuracy\n    nb_eval_steps += 1\n\n  print(\"Validation Accuracy: {}\".format(eval_accuracy/nb_eval_steps))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,8))\nplt.title(\"Training loss\")\nplt.xlabel(\"Batch\")\nplt.ylabel(\"Loss\")\nplt.plot(train_loss_set)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Test Data Preparation"},{"metadata":{"trusted":true},"cell_type":"code","source":"#creates list of sentences and labels\nsentences = all_test.question_text.values\nqids = all_test.qid.values\n\n#user tokenizer to convert sentences into tokenizer\ninput_ids  = [tokenizer.encode(sent,add_special_tokens=True,max_length=MAX_LEN) for sent in sentences]\n\n# Pad our input tokens\ninput_ids = pad_sequences(input_ids, maxlen=MAX_LEN, dtype=\"long\", truncating=\"post\", padding=\"post\")\n\n# Create attention masks\nattention_masks = []\n\n# Create a mask of 1s for each token followed by 0s for padding\nfor seq in input_ids:\n  seq_mask = [float(i>0) for i in seq]\n  attention_masks.append(seq_mask)\n\nprediction_inputs = torch.tensor(input_ids)\nprediction_masks = torch.tensor(attention_masks)\n\nbatch_size = 16 \n\n\nprediction_data = TensorDataset(prediction_inputs, prediction_masks,)\nprediction_sampler = SequentialSampler(prediction_data)\nprediction_dataloader = DataLoader(prediction_data, sampler=prediction_sampler, batch_size=batch_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Prediction on testing set\nmodel.eval()\nflat_pred = []\nfor batch in prediction_dataloader:\n  # Add batch to GPU\n  batch = tuple(t.to(device) for t in batch)\n  # Unpack the inputs from our dataloader\n  b_input_ids, b_input_mask = batch\n  # Telling the model not to compute or store gradients, saving memory and speeding up prediction\n  with torch.no_grad():\n    # Forward pass, calculate logit predictions\n    outputs =  model(b_input_ids,token_type_ids=None, attention_mask=b_input_mask)\n    logits = outputs[0]\n    logits = logits.detach().cpu().numpy() \n    flat_pred.extend(np.argmax(logits, axis=1).flatten())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction on Test Set"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_df = pd.DataFrame(columns=[\"qid\",\"prediction\"])\npred_df[\"qid\"] = qids\npred_df[\"prediction\"]  = flat_pred\npred_df.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}