{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\nfrom sklearn.model_selection import train_test_split\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom datetime import datetime","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"!pip install bert-tensorflow","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import bert\nfrom bert import run_classifier\nfrom bert import optimization\nfrom bert import tokenization","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow import keras\nimport os\nimport re\n\n# Load all files from a directory in a DataFrame.\ndef load_directory_data(directory):\n  data = {}\n  data[\"sentence\"] = []\n  data[\"sentiment\"] = []\n  for file_path in os.listdir(directory):\n    with tf.gfile.GFile(os.path.join(directory, file_path), \"r\") as f:\n      data[\"sentence\"].append(f.read())\n      data[\"sentiment\"].append(re.match(\"\\d+_(\\d+)\\.txt\", file_path).group(1))\n  return pd.DataFrame.from_dict(data)\n\n# Merge positive and negative examples, add a polarity column and shuffle.\ndef load_dataset(directory):\n  pos_df = load_directory_data(os.path.join(directory, \"pos\"))\n  neg_df = load_directory_data(os.path.join(directory, \"neg\"))\n  pos_df[\"polarity\"] = 1\n  neg_df[\"polarity\"] = 0\n  return pd.concat([pos_df, neg_df]).sample(frac=1).reset_index(drop=True)\n\n# Download and process the dataset files.\ndef download_and_load_datasets(force_download=False):\n  dataset = tf.keras.utils.get_file(\n      fname=\"aclImdb.tar.gz\", \n      origin=\"http://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\", \n      extract=True)\n  \n  train_df = load_dataset(os.path.join(os.path.dirname(dataset), \n                                       \"aclImdb\", \"train\"))\n  test_df = load_dataset(os.path.join(os.path.dirname(dataset), \n                                      \"aclImdb\", \"test\"))\n  \n  return train_df, test_df\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndf = pd.read_csv(\"../input/train.csv\")\ntrain, test = train_df, val_df = train_test_split(df, test_size= 0.4, random_state=2018)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.sample(350000)\ntest = test.sample(50000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_COLUMN = 'question_text'\nLABEL_COLUMN = 'target'\n# label_list is the list of labels, i.e. True, False or 0, 1 or 'dog', 'cat'\nlabel_list = [0, 1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use the InputExample class from BERT's run_classifier code to create examples from the data\ntrain_InputExamples = train.apply(lambda x: bert.run_classifier.InputExample(guid=None, # Globally unique ID for bookkeeping, unused in this example\n                                                                   text_a = x[DATA_COLUMN], \n                                                                   text_b = None, \n                                                                   label = x[LABEL_COLUMN]), axis = 1)\n\ntest_InputExamples = test.apply(lambda x: bert.run_classifier.InputExample(guid=None, \n                                                                   text_a = x[DATA_COLUMN], \n                                                                   text_b = None, \n                                                                   label = x[LABEL_COLUMN]), axis = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This is a path to an uncased (all lowercase) version of BERT\nBERT_MODEL_HUB = \"https://tfhub.dev/google/bert_uncased_L-12_H-768_A-12/1\"\n\ndef create_tokenizer_from_hub_module():\n  \"\"\"Get the vocab file and casing info from the Hub module.\"\"\"\n  with tf.Graph().as_default():\n    bert_module = hub.Module(BERT_MODEL_HUB)\n    tokenization_info = bert_module(signature=\"tokenization_info\", as_dict=True)\n    with tf.Session() as sess:\n      vocab_file, do_lower_case = sess.run([tokenization_info[\"vocab_file\"],\n                                            tokenization_info[\"do_lower_case\"]])\n      \n  return bert.tokenization.FullTokenizer(\n      vocab_file=vocab_file, do_lower_case=do_lower_case)\n\ntokenizer = create_tokenizer_from_hub_module()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer.tokenize(\"This here's an example of using the BERT tokenizer\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We'll set sequences to be at most 128 tokens long.\nMAX_SEQ_LENGTH = 128\n# Convert our train and test features to InputFeatures that BERT understands.\ntrain_features = bert.run_classifier.convert_examples_to_features(train_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)\ntest_features = bert.run_classifier.convert_examples_to_features(test_InputExamples, label_list, MAX_SEQ_LENGTH, tokenizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(is_predicting, input_ids, input_mask, segment_ids, labels,\n                 num_labels):\n  \"\"\"Creates a classification model.\"\"\"\n\n  bert_module = hub.Module(\n      BERT_MODEL_HUB,\n      trainable=True)\n  bert_inputs = dict(\n      input_ids=input_ids,\n      input_mask=input_mask,\n      segment_ids=segment_ids)\n  bert_outputs = bert_module(\n      inputs=bert_inputs,\n      signature=\"tokens\",\n      as_dict=True)\n\n  # Use \"pooled_output\" for classification tasks on an entire sentence.\n  # Use \"sequence_outputs\" for token-level output.\n  output_layer = bert_outputs[\"pooled_output\"]\n\n  hidden_size = output_layer.shape[-1].value\n\n  # Create our own layer to tune for politeness data.\n  output_weights = tf.get_variable(\n      \"output_weights\", [num_labels, hidden_size],\n      initializer=tf.truncated_normal_initializer(stddev=0.02))\n\n  output_bias = tf.get_variable(\n      \"output_bias\", [num_labels], initializer=tf.zeros_initializer())\n\n  with tf.variable_scope(\"loss\"):\n\n    # Dropout helps prevent overfitting\n    output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)\n\n    logits = tf.matmul(output_layer, output_weights, transpose_b=True)\n    logits = tf.nn.bias_add(logits, output_bias)\n    log_probs = tf.nn.log_softmax(logits, axis=-1)\n\n    # Convert labels into one-hot encoding\n    one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)\n\n    predicted_labels = tf.squeeze(tf.argmax(log_probs, axis=-1, output_type=tf.int32))\n    # If we're predicting, we want predicted labels and the probabiltiies.\n    if is_predicting:\n      return (predicted_labels, log_probs)\n\n    # If we're train/eval, compute loss between predicted and actual label\n    per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)\n    loss = tf.reduce_mean(per_example_loss)\n    return (loss, predicted_labels, log_probs)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# model_fn_builder actually creates our model function\n# using the passed parameters for num_labels, learning_rate, etc.\ndef model_fn_builder(num_labels, learning_rate, num_train_steps,\n                     num_warmup_steps):\n  \"\"\"Returns `model_fn` closure for TPUEstimator.\"\"\"\n  def model_fn(features, labels, mode, params):  # pylint: disable=unused-argument\n    \"\"\"The `model_fn` for TPUEstimator.\"\"\"\n\n    input_ids = features[\"input_ids\"]\n    input_mask = features[\"input_mask\"]\n    segment_ids = features[\"segment_ids\"]\n    label_ids = features[\"label_ids\"]\n\n    is_predicting = (mode == tf.estimator.ModeKeys.PREDICT)\n    \n    # TRAIN and EVAL\n    if not is_predicting:\n\n      (loss, predicted_labels, log_probs) = create_model(\n        is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)\n\n      train_op = bert.optimization.create_optimizer(\n          loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu=False)\n\n      # Calculate evaluation metrics. \n      def metric_fn(label_ids, predicted_labels):\n        accuracy = tf.metrics.accuracy(label_ids, predicted_labels)\n        f1_score = tf.contrib.metrics.f1_score(\n            label_ids,\n            predicted_labels)\n        auc = tf.metrics.auc(\n            label_ids,\n            predicted_labels)\n        recall = tf.metrics.recall(\n            label_ids,\n            predicted_labels)\n        precision = tf.metrics.precision(\n            label_ids,\n            predicted_labels) \n        true_pos = tf.metrics.true_positives(\n            label_ids,\n            predicted_labels)\n        true_neg = tf.metrics.true_negatives(\n            label_ids,\n            predicted_labels)   \n        false_pos = tf.metrics.false_positives(\n            label_ids,\n            predicted_labels)  \n        false_neg = tf.metrics.false_negatives(\n            label_ids,\n            predicted_labels)\n        return {\n            \"eval_accuracy\": accuracy,\n            \"f1_score\": f1_score,\n            \"auc\": auc,\n            \"precision\": precision,\n            \"recall\": recall,\n            \"true_positives\": true_pos,\n            \"true_negatives\": true_neg,\n            \"false_positives\": false_pos,\n            \"false_negatives\": false_neg\n        }\n\n      eval_metrics = metric_fn(label_ids, predicted_labels)\n\n      if mode == tf.estimator.ModeKeys.TRAIN:\n        return tf.estimator.EstimatorSpec(mode=mode,\n          loss=loss,\n          train_op=train_op)\n      else:\n          return tf.estimator.EstimatorSpec(mode=mode,\n            loss=loss,\n            eval_metric_ops=eval_metrics)\n    else:\n      (predicted_labels, log_probs) = create_model(\n        is_predicting, input_ids, input_mask, segment_ids, label_ids, num_labels)\n\n      predictions = {\n          'probabilities': log_probs,\n          'labels': predicted_labels\n      }\n      return tf.estimator.EstimatorSpec(mode, predictions=predictions)\n\n  # Return the actual model function in the closure\n  return model_fn\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compute train and warmup steps from batch size\n# These hyperparameters are copied from this colab notebook (https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb)\nBATCH_SIZE = 32\nLEARNING_RATE = 2e-5\nNUM_TRAIN_EPOCHS = 3.0\n# Warmup is a period of time where hte learning rate \n# is small and gradually increases--usually helps training.\nWARMUP_PROPORTION = 0.1\n# Model configs\nSAVE_CHECKPOINTS_STEPS = 500\nSAVE_SUMMARY_STEPS = 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compute # train and warmup steps from batch size\nnum_train_steps = int(len(train_features) / BATCH_SIZE * NUM_TRAIN_EPOCHS)\nnum_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Specify outpit directory and number of checkpoint steps to save\nrun_config = tf.estimator.RunConfig(\n    model_dir=OUTPUT_DIR,\n    save_summary_steps=SAVE_SUMMARY_STEPS,\n    save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_fn = model_fn_builder(\n  num_labels=len(label_list),\n  learning_rate=LEARNING_RATE,\n  num_train_steps=num_train_steps,\n  num_warmup_steps=num_warmup_steps)\n\nestimator = tf.estimator.Estimator(\n  model_fn=model_fn,\n  params={\"batch_size\": BATCH_SIZE})\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create an input function for training. drop_remainder = True for using TPUs.\ntrain_input_fn = bert.run_classifier.input_fn_builder(\n    features=train_features,\n    seq_length=MAX_SEQ_LENGTH,\n    is_training=True,\n    drop_remainder=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Beginning Training!')\ncurrent_time = datetime.now()\nestimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\nprint(\"Training took time \", datetime.now() - current_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_input_fn = run_classifier.input_fn_builder(\n    features=test_features,\n    seq_length=MAX_SEQ_LENGTH,\n    is_training=False,\n    drop_remainder=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"estimator.evaluate(input_fn=test_input_fn, steps=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def getPrediction(in_sentences):\n  labels = [0, 1]\n  input_examples = [run_classifier.InputExample(guid=\"\", text_a = x, text_b = None, label = 0) for x in in_sentences] # here, \"\" is just a dummy label\n  input_features = run_classifier.convert_examples_to_features(input_examples, label_list, MAX_SEQ_LENGTH, tokenizer)\n  predict_input_fn = run_classifier.input_fn_builder(features=input_features, seq_length=MAX_SEQ_LENGTH, is_training=False, drop_remainder=False)\n  predictions = estimator.predict(predict_input_fn)\n  return [(sentence, prediction['probabilities'], labels[prediction['labels']]) for sentence, prediction in zip(in_sentences, predictions)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test1 = pd.read_csv(\"../input/test.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = getPrediction(pred_sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}