{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TEXT SENTIMENT ANALYSIS USING BERT\n\nUsing Keras and Tensorflow 2","metadata":{}},{"cell_type":"markdown","source":"Applied to Quora Insincere Questions Competition Data","metadata":{}},{"cell_type":"code","source":"!pip install -q tensorflow-text\n!pip install -q tf-models-official","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nimport tensorflow_hub as hub\nimport tensorflow_text as text\nfrom tensorflow.keras import losses\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\nfrom official.nlp import optimization\n\nfrom sklearn.metrics import classification_report\n\nimport numpy as np\nimport pandas as pd\n\nimport gc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\",index_col=0)\ndf.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Shuffle df\ndf = df.sample(frac=1,random_state=21)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.target.value_counts())\ntarget_mean = df.target.mean()\nprint('Pct Insincere = {:.2%}'.format(target_mean))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Question Examples by Label","metadata":{}},{"cell_type":"code","source":"test_examples = df.groupby(\"target\").head(10)\n\nwith pd.option_context('display.max_colwidth', 400):\n    display(test_examples)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Tensorflow Datasets","metadata":{}},{"cell_type":"code","source":"train_ds      = tf.data.Dataset.from_tensor_slices((df.question_text.values[0::3], df.target.values[0::3])).batch(32)\nvalidation_ds = tf.data.Dataset.from_tensor_slices((df.question_text.values[1::3], df.target.values[1::3])).batch(32)\ntest_ds       = tf.data.Dataset.from_tensor_slices((df.question_text.values[2::3], df.target.values[2::3])).batch(32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\ndef configure_dataset(dataset):\n    return dataset.cache().prefetch(buffer_size=AUTOTUNE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds      = configure_dataset(train_ds.take(6000))\nvalidation_ds = configure_dataset(validation_ds.take(300))\ntest_ds       = configure_dataset(test_ds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Load BERT Model","metadata":{}},{"cell_type":"code","source":"bert_model            = hub.KerasLayer('https://tfhub.dev/tensorflow/small_bert/bert_en_uncased_L-2_H-128_A-2/1')\nbert_preprocess_model = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_classifier_model():\n    text_input = tf.keras.layers.Input(shape=(), dtype=tf.string, name='text')\n    preprocessing_layer = hub.KerasLayer('https://tfhub.dev/tensorflow/bert_en_uncased_preprocess/3', name='preprocessing')\n    encoder_inputs = preprocessing_layer(text_input)\n    encoder = hub.KerasLayer('https://tfhub.dev/google/electra_small/2', trainable=True, name='BERT_encoder')\n    outputs = encoder(encoder_inputs)\n    net = outputs['pooled_output']\n    # New Layers\n    net = tf.keras.layers.Dropout(0.1)(net)\n    # Final Layer for Classification\n    net = tf.keras.layers.Dense(1, name='classifier')(net)\n    return tf.keras.Model(text_input, net)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"classifier_model = build_classifier_model()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"loss = tf.keras.losses.BinaryCrossentropy(from_logits=True)\nmetrics = tf.metrics.BinaryAccuracy()\n\nepochs = 1\nsteps_per_epoch = tf.data.experimental.cardinality(train_ds).numpy()\n#steps_per_epoch = min(steps_per_epoch, 200)\n\nnum_train_steps = steps_per_epoch * epochs\nnum_warmup_steps = int(0.1*num_train_steps)\n\ninit_lr = 1e-5\noptimizer = optimization.create_optimizer(init_lr=init_lr,\n                                          num_train_steps=num_train_steps,\n                                          num_warmup_steps=num_warmup_steps,\n                                          optimizer_type='adamw')\n\nclassifier_model.compile(optimizer=optimizer,\n                         loss=loss,\n                         metrics=metrics,\n                         )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = classifier_model.fit(\n    x=train_ds,\n    validation_data=validation_ds,\n    epochs=epochs,\n    #steps_per_epoch=steps_per_epoch\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate and Compare Models","metadata":{}},{"cell_type":"code","source":"y_true = np.concatenate([t.numpy() for _,t in validation_ds])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = classifier_model.predict(validation_ds) > 0\nprint(classification_report(y_true, y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Export Model and Score New Texts","metadata":{}},{"cell_type":"code","source":"export_dir = \"./qic_bert\"\nclassifier_model.save(export_dir, include_optimizer=False)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"disk_model = tf.saved_model.load(export_dir)\nprint(\"Loaded model from disk\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_string_labels(predicted_scores_batch):\n    predicted_int_labels = (predicted_scores_batch.numpy() > 0).astype(int)\n    predicted_labels = tf.gather(['sincere','insincere'], predicted_int_labels)\n    return predicted_labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_scores = disk_model(tf.constant(test_examples.question_text))\npredicted_labels = get_string_labels(predicted_scores)\ntrue_labels = test_examples.target\nfor input, plabel, label in zip(test_examples.question_text, predicted_labels, true_labels):\n    print(\"Question: \", input)\n    print(\"Predicted label: \", plabel.numpy(), 'True Label: ', 'insincere' if label else 'sincere')\n    print()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}