{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# TEXT SENTIMENT ANALYSIS\n\nUsing Keras and Tensorflow 2","metadata":{}},{"cell_type":"markdown","source":"Applied to Quora Insincere Questions Competition Data","metadata":{}},{"cell_type":"code","source":"import os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import losses\nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\n\nfrom sklearn.metrics import classification_report\n\nimport numpy as np\nimport pandas as pd\n\nimport gc","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\",index_col=0)\ndf.info()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Shuffle df\ndf = df.sample(frac=1,random_state=21)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(df.target.value_counts())\nprint('Pct Insincere = {:.2%}'.format(df.target.mean()))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Question Examples by Label","metadata":{}},{"cell_type":"code","source":"test_examples = df.groupby(\"target\").head(10)\n\nwith pd.option_context('display.max_colwidth', 400):\n    display(test_examples)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Tensorflow Datasets","metadata":{}},{"cell_type":"code","source":"train_ds      = tf.data.Dataset.from_tensor_slices((df.question_text.values[0::3], df.target.values[0::3])).batch(32)\nvalidation_ds = tf.data.Dataset.from_tensor_slices((df.question_text.values[1::3], df.target.values[1::3])).batch(32)\ntest_ds       = tf.data.Dataset.from_tensor_slices((df.question_text.values[2::3], df.target.values[2::3])).batch(32)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds.cardinality()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del df\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for q, t in train_ds.take(1):\n    print(q)\n    print(t)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"for text_batch, label_batch in train_ds.take(1):\n    for i in range(10):\n        print(\"Question: \", text_batch[i])\n        print(\"Label:\", label_batch[i])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Tokenize","metadata":{}},{"cell_type":"code","source":"VOCAB_SIZE = 5000\n\nbinary_vectorize_layer = TextVectorization(\n    max_tokens=VOCAB_SIZE,\n    output_mode='binary')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 75\n\nint_vectorize_layer = TextVectorization(\n    max_tokens=VOCAB_SIZE,\n    output_mode='int',\n    output_sequence_length=MAX_SEQUENCE_LENGTH)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def binary_vectorize_text(text, label):\n    text = tf.expand_dims(text, -1)\n    return binary_vectorize_layer(text), label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def int_vectorize_text(text, label):\n    text = tf.expand_dims(text, -1)\n    return int_vectorize_layer(text), label","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Retrieve a batch (of 32 reviews and labels) from the dataset\ntext_batch, label_batch = next(iter(train_ds))\nfirst_question, first_label = text_batch[0], label_batch[0]\nprint(\"Question\", first_question)\nprint(\"Label\", first_label)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Make a text-only dataset (without labels), then call adapt\ntrain_text = train_ds.map(lambda q, t: q)\nbinary_vectorize_layer.adapt(train_text)\nint_vectorize_layer.adapt(train_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del train_text\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"foo = binary_vectorize_layer(tf.expand_dims('How did Quebec nationalists see their province as a nation in the 1960s?', -1))\nfoo","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(binary_vectorize_layer.get_vocabulary()[:30])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(binary_vectorize_layer.get_vocabulary()[-30:])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"txt = tf.expand_dims('How did Quebec nationalists see their province as a nation in the 1960s?', -1)\nint_vectorize_layer(txt)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(int_vectorize_layer.get_vocabulary()[:30])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(int_vectorize_layer.get_vocabulary()[-30:])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"'binary' vectorized question:\", \n      binary_vectorize_text(first_question, first_label)[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"'binary' vectorized question:\", \n      int_vectorize_text(first_question, first_label)[0])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"binary_train_ds = train_ds.take(15000).map(binary_vectorize_text)\nbinary_valid_ds = validation_ds.take(1000).map(binary_vectorize_text)\nbinary_test_ds  = test_ds.take(1000).map(binary_vectorize_text)\n\nint_train_ds    = train_ds.take(15000).map(int_vectorize_text)\nint_valid_ds    = validation_ds.take(1000).map(int_vectorize_text)\nint_test_ds     = test_ds.take(1000).map(int_vectorize_text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"AUTOTUNE = tf.data.AUTOTUNE\n\ndef configure_dataset(dataset):\n    return dataset.cache().prefetch(buffer_size=AUTOTUNE)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"binary_train_ds = configure_dataset(binary_train_ds)\nbinary_valid_ds = configure_dataset(binary_valid_ds)\nbinary_test_ds  = configure_dataset(binary_test_ds)\n\nint_train_ds    = configure_dataset(int_train_ds)\nint_valid_ds    = configure_dataset(int_valid_ds)\nint_test_ds     = configure_dataset(int_test_ds)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Build Sentiment Analysis Models","metadata":{}},{"cell_type":"markdown","source":"## Binary Tokenizer (Bag of Words Model)","metadata":{}},{"cell_type":"code","source":"binary_model = tf.keras.Sequential([tf.keras.layers.Dense(64, activation='relu')], name='binary_model')\nbinary_model.add(tf.keras.layers.Dropout(0.75))\nbinary_model.add(tf.keras.layers.Dense(2))\n\nbinary_model.compile(\n    loss=losses.SparseCategoricalCrossentropy(from_logits=True),\n    optimizer= tf.keras.optimizers.Adam(learning_rate = 2e-4),\n    metrics=['accuracy'])\n#history = binary_model.fit(binary_train_ds, validation_data=binary_valid_ds, epochs=10, steps_per_epoch=1500)\nhistory = binary_model.fit(binary_train_ds, validation_data=binary_valid_ds, epochs=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.DataFrame(history.history).style.background_gradient()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Linear model on binary vectorized data:\")\nprint(binary_model.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Convolution Net on Integer Tokenizer","metadata":{}},{"cell_type":"code","source":"def create_model(vocab_size, num_labels):\n    model = tf.keras.Sequential([\n        tf.keras.layers.Embedding(vocab_size, 64, mask_zero=True),\n        tf.keras.layers.Conv1D(64, 5, padding=\"valid\", activation=\"relu\", strides=2),\n        tf.keras.layers.GlobalMaxPooling1D(),\n        tf.keras.layers.Dense(num_labels)\n    ])\n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# vocab_size is VOCAB_SIZE + 1 since 0 is used additionally for padding.\nint_model = create_model(vocab_size=VOCAB_SIZE + 1, num_labels=2)\nint_model.compile(\n    loss=losses.SparseCategoricalCrossentropy(from_logits=True),\n    optimizer='adam',\n    metrics=['accuracy'])\nhistory = int_model.fit(int_train_ds, validation_data=int_valid_ds, epochs=1)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"ConvNet model on int vectorized data:\")\nprint(int_model.summary())","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Evaluate and Compare Models","metadata":{}},{"cell_type":"code","source":"binary_loss, binary_accuracy = binary_model.evaluate(binary_test_ds)\nint_loss, int_accuracy = int_model.evaluate(int_test_ds)\n\nprint(\"Binary model accuracy: {:2.2%}\".format(binary_accuracy))\nprint(\"Int model accuracy: {:2.2%}\".format(int_accuracy))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_true = np.concatenate([t.numpy() for _,t in binary_test_ds])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.argmax(binary_model.predict(binary_test_ds),1)\nprint(classification_report(y_true, y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred = np.argmax(int_model.predict(int_test_ds),1)\nprint(classification_report(y_true, y_pred))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"export_model = tf.keras.Sequential(\n    [int_vectorize_layer, int_model,\n     tf.keras.layers.Activation('sigmoid')])\n\nexport_model.compile(\n    loss=losses.SparseCategoricalCrossentropy(from_logits=False),\n    optimizer='adam',\n    metrics=['accuracy'])\n\n\nloss, accuracy = export_model.evaluate(test_ds.take(100))\nprint(\"Accuracy: {:2.2%}\".format(binary_accuracy))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Export Model and Score New Texts","metadata":{}},{"cell_type":"code","source":"def get_string_labels(predicted_scores_batch):\n    predicted_int_labels = tf.argmax(predicted_scores_batch, axis=1)\n    predicted_labels = tf.gather(['sincere','insincere'], predicted_int_labels)\n    return predicted_labels","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"predicted_scores = export_model.predict(test_examples.question_text)\npredicted_labels = get_string_labels(predicted_scores)\ntrue_labels = test_examples.target\nfor input, plabel, label in zip(test_examples.question_text, predicted_labels, true_labels):\n    print(\"Question: \", input)\n    print(\"Predicted label: \", plabel.numpy(), 'True Label: ', 'insincere' if label else 'sincere')\n    print()","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}