{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Sentence Classification with GloVe embeddings on Bi-LSTM Networks\n\n\n**Data Set:** \"Quora insincere questions\" \n\n**Pre_trained Embeddings:** GloVe Embedding"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# libraries\nimport os\nimport re \nimport gc\nimport numpy as np \nimport pandas as pd\npd.set_option('display.max_colwidth',None)\n\n# scikit\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.preprocessing import LabelEncoder\nfrom sklearn import metrics\n\n# gensim\nimport gensim\nfrom gensim.models import KeyedVectors\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Process the data sets\n\ndef load_datasets():\n    pd.set_option('display.max_colwidth',None)\n    train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\n    test  = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\n    return train, test\n\ntrain, test = load_datasets()\ndisplay(train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\n\ndef clean_data(data):\n    tag=re.compile(r'[0-9]+')\n    data=tag.sub(r'',data)\n    punct_tag=re.compile(r'[^\\w\\s]')\n    data=punct_tag.sub(r'',data)\n    tag=re.compile(r'\\s+')\n    data=tag.sub(r' ',data)\n    red_tag=re.compile(r'[?<=(  )\\\\]|[&&|\\|\\|-]')\n    data=red_tag.sub(r' ',data)\n    return \"\".join(data)\n    \ntrain['question_text'] = train['question_text'].apply(lambda x: clean_data(x))\ntest['question_text']  = test['question_text'].apply(lambda x: clean_data(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n#stemmimng the text\nfrom nltk.stem.porter import PorterStemmer\nfrom nltk.stem import *\n\ndef stem_corpus(data):\n    stemmer = PorterStemmer()\n    out_data=\"\"\n    for words in data:\n        out_data+= stemmer.stem(words)\n    return out_data\n\n\ntrain['question_text'] = train['question_text'].apply(lambda x: stem_corpus(x))\ntest['question_text']  = test['question_text'].apply(lambda x: stem_corpus(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Converting uppercase letters to lowercase\n\ndef convert_2lowercase(data):\n    data =[string.lower() for string in data if string.isupper]\n    return ''.join(data)\n\ntrain['question_text'] = train['question_text'].apply(lambda z: convert_2lowercase(z))\ntest['question_text']  = test['question_text'].apply(lambda z: convert_2lowercase(z))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Let's have a look at the clean and preprocessed data sets \ntrain.to_csv(\"train_data.csv\",sep=\",\",index=False)\ntest.to_csv(\"test_data.csv\",sep=\",\",index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Split the train and validation data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# keras-tf 2.0\nimport tensorflow as tf\nfrom tensorflow import keras","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# keras-tf 2.0\nfrom tensorflow.keras import regularizers\nfrom keras import layers\nfrom tensorflow.keras.layers import Input, Embedding, Bidirectional,LSTM,Dense,Flatten,Conv2D,Conv1D,GlobalMaxPooling1D,Concatenate,TimeDistributed\nfrom keras.models import Sequential, Model\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils import to_categorical\nfrom keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom keras.utils import plot_model\nfrom keras.wrappers.scikit_learn import KerasClassifier","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the input features\n\nX = train['question_text'] # input\ny = train['target'].values # target /label\n\nsentences_train,sentences_val,y_train,y_val = train_test_split(X,y,test_size=0.2,random_state=14)\n\n# tokenize the text corpus with keras tokenizer\ntokenizer = Tokenizer(num_words=100000)\ntokenizer.fit_on_texts(sentences_train)\nX_train = tokenizer.texts_to_sequences(sentences_train)\nX_val = tokenizer.texts_to_sequences(sentences_val)\n\n# Adding 1 because of  reserved 0 index\nvocab_size = len(tokenizer.word_index) + 2 # (in case of pre-trained embeddings it's +2)                         \nmaxlen = 128 # sentence length\n\nX_train = pad_sequences(X_train, padding='post', maxlen=maxlen)\nX_val = pad_sequences(X_val, padding='post', maxlen=maxlen)\n\nword_index = tokenizer.word_index\nnum_tokens = len(tokenizer.word_index)+2\n\nprint(\"Vocabulary Size / Unique Words in the Corpus:\",num_tokens)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Plot the history of the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# generic function to plot the train Vs validation loss/accuracy:\ndef plot_history(history):\n    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]\n    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]\n    acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' not in s]\n    val_acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' in s]\n    if len(loss_list) == 0:\n        print('Loss is missing in history')\n        return \n    ## As loss always exists\n    epochs = range(1,len(history.history[loss_list[0]]) + 1)\n    plt.figure(figsize=(25,15))\n    ## Accuracy\n    plt.subplot(2,2,1)\n    for l in acc_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training accuracy (' + str(format(history.history[l][-1],'.4f'))+')')\n    for l in val_acc_list:    \n        plt.plot(epochs, history.history[l], 'g', label='Validation accuracy (' + str(format(history.history[l][-1],'.4f'))+')')\n\n    plt.title('Training Accuracy Vs Validation Accuracy\\n')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    ## Loss\n    plt.subplot(2,2,2)\n    for l in loss_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training loss (' + str(str(format(history.history[l][-1],'.4f'))+')'))\n    for l in val_loss_list:\n        plt.plot(epochs, history.history[l], 'g', label='Validation loss (' + str(str(format(history.history[l][-1],'.4f'))+')'))\n    \n    plt.title('Training Loss Vs Validation Loss\\n')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generic function to plot the confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"def conf_matrix(actual, prediction, model_name):\n    cm_array=metrics.confusion_matrix(actual,prediction,labels=[0,1])\n    sns.set_context(\"notebook\", font_scale=1.1)\n    plt.figure(figsize=(5,5))\n    sns.heatmap(cm_array,annot=True, fmt='.0f',xticklabels=['Sincere','Insincere'],yticklabels=['Sincere','Insincere'])\n    plt.ylabel('True\\n')\n    plt.xlabel('Predicted\\n')\n    plt.title(model_name)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Using the Glove word embeddding"},{"metadata":{"trusted":true},"cell_type":"code","source":"import io\nfrom tqdm import tqdm\nimport zipfile\n\n## make a dict mapping words (strings) to their NumPy vector representation:\nembeddings_index = {}\n\nwith zipfile.ZipFile(\"../input/quora-insincere-questions-classification/embeddings.zip\") as zf:\n    with io.TextIOWrapper(zf.open(\"glove.840B.300d/glove.840B.300d.txt\"), encoding=\"utf-8\") as f:\n        for line in tqdm(f):\n            word, coefs = line.split(maxsplit=1)\n            coefs = np.fromstring(coefs, dtype=float, sep=\" \")\n            embeddings_index[word] = coefs\n            \n            \nprint(\"Found %s word vectors.\" % len(embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Designing Embedding Matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"## prepare a corresponding embedding matrix that we can use in a Keras Embedding layer. \n## It's a simple NumPy matrix where entry at index i is the pre-trained vector for the word of index i in our vectorizer's vocabulary.\n\nword_index=tokenizer.word_index\nnum_tokens = len(tokenizer.word_index)+ 2\nembedding_dim = 300\n\nhits = 0\nmisses = 0\n\n# Prepare embedding matrix\nembedding_matrix = np.zeros((num_tokens, embedding_dim))\n\nfor word, i in word_index.items():\n    try:\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[i] = embedding_vector\n            hits += 1\n        else:\n            misses += 1\n    except:\n        misses += 1\n        \nprint(\"Converted %d words (%d misses)\" % (hits, misses))\n\n\n#load the pre-trained word embeddings matrix into an Embedding layer.\nembedding_layer = Embedding(\n    num_tokens,\n    embedding_dim,\n    embeddings_initializer=keras.initializers.Constant(embedding_matrix),\n    trainable=False\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_dim = 300\n\n\n# Input for variable-length sequences of integers\ninputs = keras.Input(shape=(None,), dtype=\"int32\")\n# Embed each integer in a 128-dimensional vector\nx = layers.Embedding(num_tokens,embedding_dim,weights=[embedding_matrix],input_length=maxlen,trainable=False)(inputs)\n# Add 2 bidirectional LSTMs\nx = layers.Bidirectional(layers.LSTM(64,return_sequences=True))(x)\nx = layers.Bidirectional(layers.LSTM(64,return_sequences=False))(x)\nx = layers.Dense(64, activation=\"relu\")(x)\nx = layers.Dropout(0.5)(x)\n# Add a classifier\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\nmodel = keras.Model(inputs, outputs)\nmodel.summary()\n\n\nopt = Adam(learning_rate=0.01)\n\n\n# defining the call backs\nearly_stopping=tf.keras.callbacks.EarlyStopping(\n                                                monitor=\"val_loss\",\n                                                patience=3,\n                                                mode=\"min\",\n                                                restore_best_weights=True\n                                              )\n### Now reducing the learning rate when the model is not improvinig \nreduce_lr=tf.keras.callbacks.ReduceLROnPlateau(\n                                                monitor=\"val_loss\",\n                                                factor=0.2,\n                                                patience=2,\n                                                verbose=1,\n                                                mode=\"auto\"\n                                            )\n\nmy_callbacks=[early_stopping,reduce_lr]\n\nmodel.compile(optimizer=opt,loss='binary_crossentropy',metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Validate the fit "},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nepochs=2\n\nfor e in range(epochs):\n    history = model.fit(X_train,y_train,batch_size=256,epochs=5,validation_split=0.2,callbacks=my_callbacks)\n    pred_val_y = model.predict([X_val], batch_size=512, verbose=1)\n    best_thresh = 0.5\n    best_score = 0.0\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        score = metrics.f1_score(y_val, (pred_val_y>thresh).astype(int))\n        if score > best_score:\n            best_thresh = thresh\n            best_score = score\n    print(\"Val F1 Score: {:.6f}\".format(best_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_thresh","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"best_score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot history"},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metrics on Validation Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_y_val = (pred_val_y>best_thresh).astype(int)\nprint(\"Metrics\\n\")\nprint(metrics.classification_report(y_val,pred_y_val))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Confusion Matrix on Validation Data"},{"metadata":{"trusted":true},"cell_type":"code","source":"conf_matrix(y_val,pred_y_val,'Bi-RNN Model Validation Scores\\n')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Prediction - Glove"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df    = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\ntest_input = test_df['question_text']\nids        = test_df['qid']\ntest_input = np.array(test_input)\n\ntokenizer = Tokenizer(num_words=100000)\ntokenizer.fit_on_texts(test_input)\nsequences = tokenizer.texts_to_sequences(test_input)\ntest_sequences = pad_sequences(sequences, maxlen=128,padding='post')\nindices = np.arange(test_sequences.shape[0])\ntest_padded = test_sequences[indices]\ntest_padded.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Final Submission - Glove - BiLSTM"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_predictions = model.predict(test_padded)\npredictions = (test_predictions>best_thresh).astype(int)\npredictions = predictions.flatten()\n\nout_df = pd.DataFrame({'qid':ids,'prediction':predictions})\npd.set_option('display.max_colwidth',None)\n# submission file\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"out_df.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}