{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install bert-for-tf2\n!pip install sentencepiece","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    %tensorflow_version 2.x\nexcept Exception:\n    pass\nimport tensorflow as tf\n\nimport tensorflow_hub as hub\n\nfrom tensorflow.keras import layers\nimport bert","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_df = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\n\nquestion_df.isnull().values.any()\n\nquestion_df.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It's painfully slow to operate on the original dataset. For the sake of efficiency, we only consider a subset of the entire question dataset.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"question_df = question_df[:50000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_df.info","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\ndef preprocess_text(sen):\n    # Removing html tags\n    sentence = remove_tags(sen)\n\n    # Remove punctuations and numbers\n    sentence = re.sub('[^a-zA-Z]', ' ', sentence)\n\n    # Single character removal\n    sentence = re.sub(r\"\\s+[a-zA-Z]\\s+\", ' ', sentence)\n\n    # Removing multiple spaces\n    sentence = re.sub(r'\\s+', ' ', sentence)\n\n    return sentence\n\nTAG_RE = re.compile(r'<[^>]+>')\n\ndef remove_tags(text):\n    return TAG_RE.sub('', text)\n\nquestions = []\nsentences = list(question_df['question_text'])\nfor sen in sentences:\n    questions.append(preprocess_text(sen))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(question_df.columns.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_df.drop(\"qid\", axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(question_df.columns.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"question_df.target.unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# text tokenization using the BERT tokenizer\nBertTokenizer = bert.bert_tokenization.FullTokenizer\n\nbert_layer = hub.KerasLayer(\"https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1\",\n                            trainable=False)\nvocabulary_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\nto_lower_case = bert_layer.resolved_object.do_lower_case.numpy()\ntokenizer = BertTokenizer(vocabulary_file, to_lower_case)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer.tokenize(\"don't be so judgmental\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer.convert_tokens_to_ids(tokenizer.tokenize(\"dont be so judgmental\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tokenize_questions(questions):\n    return tokenizer.convert_tokens_to_ids(tokenizer.tokenize(questions))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenized_questions = [tokenize_questions(question) for question in questions]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = question_df['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = np.array(target)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_with_len = [[question, target[i], len(question)]\n                 for i, question in enumerate(tokenized_questions)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nrandom.shuffle(questions_with_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"questions_with_len.sort(key=lambda x: x[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sorted_questions_labels = [(question_lab[0], question_lab[1]) for question_lab in questions_with_len]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"processed_dataset = tf.data.Dataset.from_generator(lambda: sorted_questions_labels, output_types=(tf.int32, tf.int32))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 32\nbatched_dataset = processed_dataset.padded_batch(BATCH_SIZE, padded_shapes=((None, ), ()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"next(iter(batched_dataset))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\n\nTOTAL_BATCHES = math.ceil(len(sorted_questions_labels) / BATCH_SIZE)\nTEST_BATCHES = TOTAL_BATCHES // 10\nbatched_dataset.shuffle(TOTAL_BATCHES)\ntest_data = batched_dataset.take(TEST_BATCHES)\ntrain_data = batched_dataset.skip(TEST_BATCHES)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CNN\n\nThe neural net structure is copied from [the colab notebook](https://colab.research.google.com/drive/12noBxRkrZnIkHqvmdfFW2TGdOXFtNePM#scrollTo=VxONsFVHkFLU)\n\nThe first layer is an embedding layer that  is initialized with random weights and will learn an embedding for all of the words in the training dataset.                                                                                                                                                                                                                                    \n\nWe have 3 hidden layers with \"relu\" activation function.\n\nThe first layer has sliding window of size 2.\nThe second layer has sliding window of size 3.\nThe third layer has sliding window of size 4.\n\nThen we have a max pooling layer.\n\nThen we have a densely connected layer.\n\nThe dropout rate is 0.2.\n\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class TEXT_MODEL(tf.keras.Model):\n    \n    def __init__(self,\n                 vocabulary_size,\n                 embedding_dimensions=128,\n                 cnn_filters=50,\n                 dnn_units=512,\n                 model_output_classes=2,\n                 dropout_rate=0.1,\n                 training=False,\n                 name=\"text_model\"):\n        super(TEXT_MODEL, self).__init__(name=name)\n        \n        self.embedding = layers.Embedding(vocabulary_size,\n                                          embedding_dimensions)\n        self.cnn_layer1 = layers.Conv1D(filters=cnn_filters,\n                                        kernel_size=2,\n                                        padding=\"valid\",\n                                        activation=\"relu\")\n        self.cnn_layer2 = layers.Conv1D(filters=cnn_filters,\n                                        kernel_size=3,\n                                        padding=\"valid\",\n                                        activation=\"relu\")\n        self.cnn_layer3 = layers.Conv1D(filters=cnn_filters,\n                                        kernel_size=4,\n                                        padding=\"valid\",\n                                        activation=\"relu\")\n        self.pool = layers.GlobalMaxPool1D()\n        \n        self.dense_1 = layers.Dense(units=dnn_units, activation=\"relu\")\n        self.dropout = layers.Dropout(rate=dropout_rate)\n        if model_output_classes == 2:\n            self.last_dense = layers.Dense(units=1,\n                                           activation=\"sigmoid\")\n        else:\n            self.last_dense = layers.Dense(units=model_output_classes,\n                                           activation=\"softmax\")\n    \n    def call(self, inputs, training):\n        l = self.embedding(inputs)\n        l_1 = self.cnn_layer1(l) \n        l_1 = self.pool(l_1) \n        l_2 = self.cnn_layer2(l) \n        l_2 = self.pool(l_2)\n        l_3 = self.cnn_layer3(l)\n        l_3 = self.pool(l_3) \n        \n        concatenated = tf.concat([l_1, l_2, l_3], axis=-1) # (batch_size, 3 * cnn_filters)\n        concatenated = self.dense_1(concatenated)\n        concatenated = self.dropout(concatenated, training)\n        model_output = self.last_dense(concatenated)\n        \n        return model_output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"VOCAB_LENGTH = len(tokenizer.vocab)\nEMB_DIM = 200\nCNN_FILTERS = 100\nDNN_UNITS = 256\nOUTPUT_CLASSES = 2\n\nDROPOUT_RATE = 0.2\n#  hyperparameter that defines the number times that the learning algorithm will work through the entire training dataset.\nNB_EPOCHS = 10","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"text_model = TEXT_MODEL(vocabulary_size=VOCAB_LENGTH,\n                        embedding_dimensions=EMB_DIM,\n                        cnn_filters=CNN_FILTERS,\n                        dnn_units=DNN_UNITS,\n                        model_output_classes=OUTPUT_CLASSES,\n                        dropout_rate=DROPOUT_RATE)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if OUTPUT_CLASSES == 2:\n    text_model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n              optimizer=tf.keras.optimizers.Adam(1e-4),\n              metrics=['accuracy'])\nelse:\n    text_model.compile(loss=\"sparse_categorical_crossentropy\",\n                       optimizer=\"adam\",\n                       metrics=[\"sparse_categorical_accuracy\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"text_model.fit(train_data, epochs=NB_EPOCHS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loss, test_acc = text_model.evaluate(test_data)\n\nprint('Test Loss: {}'.format(test_loss))\nprint('Test Accuracy: {}'.format(test_acc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Accuracy on the test dataset now is 96.5%.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Try a different dropout rate of 0.5.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DROPOUT_RATE = 0.5\ntext_model = TEXT_MODEL(vocabulary_size=VOCAB_LENGTH,\n                        embedding_dimensions=EMB_DIM,\n                        cnn_filters=CNN_FILTERS,\n                        dnn_units=DNN_UNITS,\n                        model_output_classes=OUTPUT_CLASSES,\n                        dropout_rate=DROPOUT_RATE)\n\ntext_model.compile(loss=\"binary_crossentropy\",\n                       optimizer=\"adam\",\n                       metrics=[\"accuracy\"])\ntext_model.fit(train_data, epochs=NB_EPOCHS)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loss, test_acc = text_model.evaluate(test_data)\n\nprint('Test Loss: {}'.format(test_loss))\nprint('Test Accuracy: {}'.format(test_acc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Test accuracy is 80% now -- seems like CNN models are not robust and sensitive to drop out rate.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#param_grid = dict(num_filters=[32, 64, 128],\n #                 kernel_size=[3, 5, 7],\n  #                vocab_size=[5000], \n   #               embedding_dim=[50],\n    #              maxlen=[100])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Future work: hyperparameter optim.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## RNN Models\nI referenced the models in [Text classification with a RNN](https://www.tensorflow.org/tutorials/text/text_classification_rnn)","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### One Bidirectional LSTM Layer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#VOCAB_LENGTH = len(tokenizer.vocab)\n#EMB_DIM = 200\n#CNN_FILTERS = 100\n#RNN_UNITS = 256\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.Embedding(VOCAB_LENGTH, 64),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)),\n    tf.keras.layers.Dense(64, activation='relu'),\n    tf.keras.layers.Dense(1)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n              optimizer=tf.keras.optimizers.Adam(1e-4),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_data, epochs=10,\n                    validation_data=test_data, \n                    validation_steps=30)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loss, test_acc = model.evaluate(test_data)\n\nprint('Test Loss: {}'.format(test_loss))\nprint('Test Accuracy: {}'.format(test_acc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Test Accuracy now is 92.5%. This is slightly worse than our CNN model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\n\ndef plot_graphs(history, metric):\n  plt.plot(history.history[metric])\n  plt.plot(history.history['val_'+metric], '')\n  plt.xlabel(\"Epochs\")\n  plt.ylabel(metric)\n  plt.legend([metric, 'val_'+metric])\n  plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_graphs(history, 'accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_graphs(history, 'loss')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stacked two Bidirectional LSTM layers","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.Embedding(VOCAB_LENGTH, 64),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,  return_sequences=True)),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n    tf.keras.layers.Dense(64, activation='relu'),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(1)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n              optimizer=tf.keras.optimizers.Adam(1e-4),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_data, epochs=10,\n                    validation_data=test_data,\n                    validation_steps=30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loss, test_acc = model.evaluate(test_data)\n\nprint('Test Loss: {}'.format(test_loss))\nprint('Test Accuracy: {}'.format(test_acc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Accuracy now is 95.6%. This is a tiny bit better than our RNN model with only 1 LSTM layer.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_graphs(history, 'accuracy')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_graphs(history, 'loss')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stacked Three Bidirectional LSTM Layers","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Intermediate RNN layers should return full sequence of outputs; 3D tensor by specifying return_sequences=True.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.Embedding(VOCAB_LENGTH, 64),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,  return_sequences=True)),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64, return_sequences=True)),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(32)),\n    tf.keras.layers.Dense(64, activation='relu'),\n    tf.keras.layers.Dropout(0.5),\n    tf.keras.layers.Dense(1)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n              optimizer=tf.keras.optimizers.Adam(1e-4),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_data, epochs=10,\n                    validation_data=test_data,\n                    validation_steps=30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loss, test_acc = model.evaluate(test_data)\n\nprint('Test Loss: {}'.format(test_loss))\nprint('Test Accuracy: {}'.format(test_acc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Accuracy now is 94.8%. This is not better than our RNN model with only 2 LSTM layer.\n\nAdding network depths does not seem ideal in improving performances.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Playing with CNN+LSTM Hybrid","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.Embedding(VOCAB_LENGTH, 64),\n    tf.keras.layers.Conv1D(filters=64, kernel_size=3, padding='same', activation='relu'),\n    tf.keras.layers.MaxPooling1D(pool_size=2),\n    tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64)),\n    tf.keras.layers.Dense(64, activation='relu'),\n    tf.keras.layers.Dense(1)\n])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n              optimizer=tf.keras.optimizers.Adam(1e-4),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_data, epochs=10,\n                    validation_data=test_data,\n                    validation_steps=30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loss, test_acc = model.evaluate(test_data)\n\nprint('Test Loss: {}'.format(test_loss))\nprint('Test Accuracy: {}'.format(test_acc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Stacked Simple LSTMs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"model = tf.keras.Sequential([\n    tf.keras.layers.Embedding(VOCAB_LENGTH, 64),\n    tf.keras.layers.LSTM(200, activation='relu', return_sequences=True, input_shape=(1, 2)),\n    tf.keras.layers.LSTM(100, activation='relu', return_sequences=True),\n    tf.keras.layers.LSTM(50, activation='relu', return_sequences=True),\n    tf.keras.layers.LSTM(25, activation='relu'),\n    tf.keras.layers.Dense(20, activation='relu'),\n    tf.keras.layers.Dense(10, activation='relu'),\n    tf.keras.layers.Dense(1)\n   \n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),\n              optimizer=tf.keras.optimizers.Adam(1e-4),\n              metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_data, epochs=10,\n                    validation_data=test_data,\n                    validation_steps=30)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_loss, test_acc = model.evaluate(test_data)\n\nprint('Test Loss: {}'.format(test_loss))\nprint('Test Accuracy: {}'.format(test_acc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have a pretty high accuracy of 96.6%. Simple LSTMs perform better than Bilateral LSTMs in this case, and seems to be more robust than our CNN model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}