{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Files supplied by Quora dataset:\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-21T09:34:25.105875Z","iopub.execute_input":"2021-06-21T09:34:25.106485Z","iopub.status.idle":"2021-06-21T09:34:25.125392Z","shell.execute_reply.started":"2021-06-21T09:34:25.106349Z","shell.execute_reply":"2021-06-21T09:34:25.124198Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os.path\n# print(os.getcwd())\nif not os.path.isfile(\"/kaggle/working/glove.840B.300d/glove.840B.300d.txt\") :\n    print(\"Extracting\")\n    !unzip -j /kaggle/input/quora-insincere-questions-classification/embeddings.zip \"glove.840B.300d/glove.840B.300d.txt\" -d \"/kaggle/working/glove.840B.300d/\"","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:34:25.127219Z","iopub.execute_input":"2021-06-21T09:34:25.127728Z","iopub.status.idle":"2021-06-21T09:35:30.970272Z","shell.execute_reply.started":"2021-06-21T09:34:25.127679Z","shell.execute_reply":"2021-06-21T09:35:30.969082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Install the attention module. There are multiple modules available, out of keras as well as in keras itself. Installing one below for demo purpuse","metadata":{}},{"cell_type":"code","source":"!pip install keras-self-attention\n!pip install attention","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:35:30.972537Z","iopub.execute_input":"2021-06-21T09:35:30.972893Z","iopub.status.idle":"2021-06-21T09:35:56.244619Z","shell.execute_reply.started":"2021-06-21T09:35:30.972855Z","shell.execute_reply":"2021-06-21T09:35:56.243822Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!ls -ltrh /kaggle/input/quora-insincere-questions-classification/ ","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:35:56.246675Z","iopub.execute_input":"2021-06-21T09:35:56.247013Z","iopub.status.idle":"2021-06-21T09:35:56.973537Z","shell.execute_reply.started":"2021-06-21T09:35:56.246979Z","shell.execute_reply":"2021-06-21T09:35:56.972714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Import all necessary modules in one go:","metadata":{}},{"cell_type":"code","source":"import os\nimport tensorflow as tf\nos.chdir('/kaggle/input/quora-insincere-questions-classification')\nimport pandas as pd\nimport keras\nfrom tensorflow.keras import layers\nfrom keras.models import Sequential\nfrom keras.layers import LSTM, Dropout, Dense, Input, Flatten\nfrom keras.layers import Bidirectional\nfrom keras.layers.embeddings import Embedding\nfrom keras.preprocessing import sequence\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.text import Tokenizer","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:35:56.975081Z","iopub.execute_input":"2021-06-21T09:35:56.975650Z","iopub.status.idle":"2021-06-21T09:36:02.973510Z","shell.execute_reply.started":"2021-06-21T09:35:56.975604Z","shell.execute_reply":"2021-06-21T09:36:02.972628Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Read the dataset in train and test dataframe. In this notebook we will use only train, and split it for training and validation. Supplied test.csv is used for competition and not useful for this attention demo","metadata":{}},{"cell_type":"code","source":"trainAll = train = pd.read_csv('train.csv')\ntest = pd.read_csv('test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:36:02.975115Z","iopub.execute_input":"2021-06-21T09:36:02.975558Z","iopub.status.idle":"2021-06-21T09:36:09.097053Z","shell.execute_reply.started":"2021-06-21T09:36:02.975515Z","shell.execute_reply":"2021-06-21T09:36:09.096053Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Have a quick look at training csv\ntrain.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:36:09.098330Z","iopub.execute_input":"2021-06-21T09:36:09.098639Z","iopub.status.idle":"2021-06-21T09:36:09.126329Z","shell.execute_reply.started":"2021-06-21T09:36:09.098610Z","shell.execute_reply":"2021-06-21T09:36:09.125355Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### Define function below to plot the history of model training output. Useful to understand if model is overfilling","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n# Visualize training history\ndef plot_history(history):\n# print(history.history.keys())\n    # summarize history for accuracy\n    plt.plot(history.history['accuracy'])\n    plt.plot(history.history['val_accuracy'])\n    plt.title('model accuracy')\n    plt.ylabel('accuracy')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper left')\n    plt.show()\n    # summarize history for loss\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('model loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'test'], loc='upper left')\n    plt.show()\n\nfrom IPython.display import Image, display\ndef plot_model(model):\n    tf.keras.utils.plot_model(model, to_file=\"/kaggle/working/model.png\", show_shapes=True, show_dtype=False, show_layer_names=True, rankdir=\"TB\", expand_nested=True)\n    display(Image(filename=\"/kaggle/working/model.png\"))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:39:22.094997Z","iopub.execute_input":"2021-06-21T09:39:22.095359Z","iopub.status.idle":"2021-06-21T09:39:22.104464Z","shell.execute_reply.started":"2021-06-21T09:39:22.095331Z","shell.execute_reply":"2021-06-21T09:39:22.103327Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Read some portion dataset for training\n#### Just take 1% of dataset for quicker training. If whole dataset to be used, then need to increase the model size.","metadata":{}},{"cell_type":"code","source":"def preprocessing_text(df):\n    df = pd.DataFrame(df)\n    for i,row in df.iterrows():\n#         print(i, row)\n        df.at[i,'question_text'] = str(row[\"question_text\"]).replace('n\\'t', ' not').replace('\\'', '').replace('\\\"', '')\n    return df['question_text']\n# preprocessing_text(X)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:36:09.139059Z","iopub.execute_input":"2021-06-21T09:36:09.139354Z","iopub.status.idle":"2021-06-21T09:36:09.149237Z","shell.execute_reply.started":"2021-06-21T09:36:09.139328Z","shell.execute_reply":"2021-06-21T09:36:09.148294Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# embedding_vecor_length = 32\ntrain = trainAll\ntrain = train.sample(frac=0.01, random_state=np.random.RandomState() )   ## TAke just 1% of training data for this exercise\nX = train[\"question_text\"]\nX = preprocessing_text(X)\ny=train[\"target\"]\n\nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 200 # max number of words in a question to use\n\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X))\nX_sequences = tokenizer.texts_to_sequences(X)\n# val_X = tokenizer.texts_to_sequences(val_X)\n# test_X = tokenizer.texts_to_sequences(test_X)\n\nX_train, X_test, y_train, y_test = train_test_split(X_sequences, y, test_size = 0.1)\nX_train = sequence.pad_sequences(X_train, maxlen=maxlen, padding='post')\nX_test = sequence.pad_sequences(X_test, maxlen=maxlen, padding='post')\nprint(X_train[:10])","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:36:09.150451Z","iopub.execute_input":"2021-06-21T09:36:09.150744Z","iopub.status.idle":"2021-06-21T09:36:11.260974Z","shell.execute_reply.started":"2021-06-21T09:36:09.150719Z","shell.execute_reply":"2021-06-21T09:36:11.259969Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Prepare embedding layer\n#### Use the glove embedding file to prepare the Embedding layer for models below","metadata":{}},{"cell_type":"code","source":"glove_file=\"/kaggle/working/glove.840B.300d/glove.840B.300d.txt\"\nimport tqdm\n\nEMBEDDING_VECTOR_LENGTH = 300 #50 # <=200\nOOV=[]\ndef construct_embedding_matrix(glove_file, word_index):\n    global OOV\n    embedding_dict = {}\n    with open(glove_file,'r') as f:\n        for line in f:\n            values=line.split(' ')\n            # get the word\n            word=values[0]\n            if word in word_index.keys():\n                # get the vector\n                try:  vector = np.asarray(values[1:], 'float32')\n                except:\n                    print(\"Error at:\", values[:5])\n                    pass\n                embedding_dict[word] = vector\n    ###  oov words (out of vacabulary words) will be mapped to 0 vectors\n\n    num_words=len(word_index)+1\n    #initialize it to 0\n    embedding_matrix=np.zeros((num_words, EMBEDDING_VECTOR_LENGTH))\n    \n    for word,i in tqdm.tqdm(word_index.items()):\n        if i < num_words:\n            vect=embedding_dict.get(word, [])\n            if len(vect)>0:\n                embedding_matrix[i] = vect[:EMBEDDING_VECTOR_LENGTH]\n            else:\n                OOV.append(word)\n                \n    print(\"OOV words.. first 100:\", OOV[:100])            \n    print(\"OOV word printed above (if any), should be handled for good results.\")\n\n    return embedding_matrix\n  \nif 'embedding_matrix' not in vars() or  embedding_matrix is None:\n    embedding_matrix =  construct_embedding_matrix(glove_file, tokenizer.word_index)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:36:11.262219Z","iopub.execute_input":"2021-06-21T09:36:11.262498Z","iopub.status.idle":"2021-06-21T09:36:51.083599Z","shell.execute_reply.started":"2021-06-21T09:36:11.262472Z","shell.execute_reply":"2021-06-21T09:36:51.082527Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Number of words not understood by glove (need more preprocesing): \", len(OOV))","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:36:51.085244Z","iopub.execute_input":"2021-06-21T09:36:51.085668Z","iopub.status.idle":"2021-06-21T09:36:51.091800Z","shell.execute_reply.started":"2021-06-21T09:36:51.085623Z","shell.execute_reply":"2021-06-21T09:36:51.090705Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from keras.initializers import Constant\nembedding=Embedding(len(tokenizer.word_index)+1, # number of unique tokens\n                    EMBEDDING_VECTOR_LENGTH, #number of features\n                    embeddings_initializer=Constant(embedding_matrix), # initialize \n                    input_length=maxlen, \n                    trainable=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:36:51.093220Z","iopub.execute_input":"2021-06-21T09:36:51.093545Z","iopub.status.idle":"2021-06-21T09:36:51.116060Z","shell.execute_reply.started":"2021-06-21T09:36:51.093515Z","shell.execute_reply":"2021-06-21T09:36:51.115312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### I will run on TPU , as i am bit impatient","metadata":{}},{"cell_type":"code","source":"tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect()\n\n# instantiate a distribution strategy\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)\n\n# instantiating the model in the strategy scope creates the model on the TPU","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:36:51.118458Z","iopub.execute_input":"2021-06-21T09:36:51.118798Z","iopub.status.idle":"2021-06-21T09:36:56.983231Z","shell.execute_reply.started":"2021-06-21T09:36:51.118745Z","shell.execute_reply":"2021-06-21T09:36:56.982176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### As we are going to build multiple models below, so let's define common layers","metadata":{}},{"cell_type":"code","source":"lstm = dropout = dense_last = flatten = epochs = opt = None\ndef rebuild_layers():\n    global lstm , dropout,  dense_last, epochs, opt , flatten\n    lstm = LSTM(20, return_sequences=True)\n    dropout = Dropout(0.05)\n    dense_last = Dense(5, kernel_regularizer=keras.regularizers.l2())\n    epochs = 7\n    opt = keras.optimizers.Adam(learning_rate=0.001)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:41:29.124696Z","iopub.execute_input":"2021-06-21T09:41:29.125223Z","iopub.status.idle":"2021-06-21T09:41:29.130311Z","shell.execute_reply.started":"2021-06-21T09:41:29.125185Z","shell.execute_reply":"2021-06-21T09:41:29.129644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## A simple LSTM model \n#### As data is very low, so just 2 LSTM layers. Model overfits quickly so adding a dropout layer too with 20% dropout","metadata":{}},{"cell_type":"code","source":"rebuild_layers()\nwith tpu_strategy.scope():\n    model = tf.keras.Sequential() # define your model normally\n    model.add(Input(shape=(maxlen,)))\n#     model.add(Embedding(max_features, embed_size))\n    model.add(embedding)\n    model.add(lstm)\n    model.add(dropout)\n    model.add(dense_last)\n    model.add(Flatten())\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])\n    print(model.summary())\n    plot_model(model)\n\nhistory = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=epochs, batch_size=64)\nplot_history(history)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:36:56.991849Z","iopub.execute_input":"2021-06-21T09:36:56.992132Z","iopub.status.idle":"2021-06-21T09:37:26.474629Z","shell.execute_reply.started":"2021-06-21T09:36:56.992105Z","shell.execute_reply":"2021-06-21T09:37:26.473567Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Note the accuracy of above model training in final epoch. For current run it was: 96.02% \n##### (Note: It may vary with each run due to presence of random sample of dataset and random dropout on each run)","metadata":{}},{"cell_type":"markdown","source":"## Bidrectional LSTM model without Attention layer","metadata":{}},{"cell_type":"code","source":"rebuild_layers()\nwith tpu_strategy.scope():\n    model = tf.keras.Sequential() # define your model normally\n    model.add(Input(shape=(maxlen,)))\n#     model.add(Embedding(max_features, embed_size))\n    model.add(embedding)\n    model.add(Bidirectional(lstm))\n    model.add(dropout)\n    model.add(dense_last)\n    model.add(Flatten())\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])\nprint(model.summary())\nplot_model(model)\n\nhistory = model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=epochs, batch_size=64)\nplot_history(history)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:47:01.526251Z","iopub.execute_input":"2021-06-21T09:47:01.526627Z","iopub.status.idle":"2021-06-21T09:47:41.420872Z","shell.execute_reply.started":"2021-06-21T09:47:01.526591Z","shell.execute_reply":"2021-06-21T09:47:41.419870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Note the accuracy of above model training in final epoch. For current run it was: 96.48%  - slightly better than non bidirectional version above","metadata":{}},{"cell_type":"markdown","source":"##### For attention code, have to use keras fucntional api instead of sequential. Because attention takes a different route in the flow as seen in the model plot below","metadata":{}},{"cell_type":"markdown","source":"## Same bidrectional model as above with Attention (Self attention) layer (for comaprision of results)","metadata":{}},{"cell_type":"code","source":"rebuild_layers()\nfrom attention import Attention\nrebuild_layers()\nwith tpu_strategy.scope():\n    model_input   = Input(shape=(maxlen,))\n    x = Embedding(len(tokenizer.word_index)+1, # number of unique tokens\n                    EMBEDDING_VECTOR_LENGTH, #number of features\n                    embeddings_initializer=Constant(embedding_matrix), # initialize \n                    input_length=maxlen, \n                    trainable=False)(model_input)\n    x = Bidirectional(LSTM(20, return_sequences=True))(x) #model_input\n    x = Attention(50)(x)\n    x = dropout(x)\n    x = dense_last(x)\n    x = Flatten()(x)\n    x = Dense(1, activation='sigmoid')(x)\n    modelAtt = keras.models.Model(model_input, x)\n    modelAtt.compile(loss='binary_crossentropy', optimizer=opt,metrics=[\"accuracy\"], experimental_run_tf_function=False) #, run_eagerly=True\n    print(modelAtt.summary())\n    plot_model(modelAtt)\n    \nhistory = modelAtt.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=epochs, batch_size=64)\nplot_history(history)","metadata":{"execution":{"iopub.status.busy":"2021-06-21T09:48:15.764438Z","iopub.execute_input":"2021-06-21T09:48:15.764828Z","iopub.status.idle":"2021-06-21T09:48:55.989651Z","shell.execute_reply.started":"2021-06-21T09:48:15.764791Z","shell.execute_reply":"2021-06-21T09:48:55.988736Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"##### Note the accuracy of above model training in final epoch. For current run it was: 97.02%  better than non attention version above","metadata":{}},{"cell_type":"markdown","source":"# It is clear from above last 2 models that, due to applying attention layer the accuracy has improved. As model has learnt faster than non attention version","metadata":{}}]}