{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Comparison of different RNN architectures\nIn this notebook, we will compare different architectures of Recurrent Neural Networks. We will use data from the competition [Jigsaw Multilingual Toxic Comment Classification](https://www.kaggle.com/c/jigsaw-multilingual-toxic-comment-classification). We will use jigsaw-toxic-comment-train.csv file and divide it into train and test datasets.<br><br>\nWe will use 50 dimensional [Glove vectors](https://nlp.stanford.edu/projects/glove/) as word embeddings.\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport random\nfrom tqdm.notebook import tqdm\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.layers import  Embedding, Input, LSTM, Dense, SimpleRNN, Bidirectional\nfrom keras.initializers import Constant\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.metrics import AUC\nfrom sklearn.metrics import accuracy_score, roc_auc_score\n\nrandom.seed(0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Hyperparameters\n\nTEST_PROPORTION = 0.2 #proportion of dataset to be used for testing\nMAX_VOCAB_SIZE = 100000 #Maximum vocabulary size\nMAX_SEQ_LEN = 128 #Maximum sequence length for comments\nEMBEDDING_SIZE = 50 #Size of word embedding vector","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"## Reading Data\n\ndata = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\nprint (\"Number of rows in data: \", data.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will only use two columns -\n1. comment_text \n2. toxic - label"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Function to split data into train and test sets \n\ndef train_test_split(df,test_prop=0.25):\n    n_rows = df.shape[0]\n    list_indices = list(range(n_rows))\n    random.shuffle(list_indices)\n    n_rows_test = int(n_rows*test_prop)\n    n_rows_train = n_rows - n_rows_test\n    df_train = df.iloc[list_indices[:n_rows_train]]\n    df_test = df.iloc[list_indices[n_rows_train:]]\n    return df_train, df_test\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Function to read Glove Embeddings\n\ndef read_glove_vecs(glove_file):\n    words = []\n    word_to_vec_map = {}\n    with open(glove_file,'r') as f:\n        for line in f:\n            line = line.strip().split()\n            words += [line[0]]\n            word_to_vec_map[line[0]] = np.array(line[1:], dtype = np.float64)\n    return words, word_to_vec_map","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_train,data_test =train_test_split(data, TEST_PROPORTION)\nprint (\"Number of rows in train dataset: \", data_train.shape)\nprint (\"Number of rows in test dataset: \", data_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Reading the Glove embeddings\n\nwords, word_to_vec_map = read_glove_vecs('../input/glove6b50dtxt/glove.6B.50d.txt')\nprint (\"Size of vocabulary: \", len(words))\nprint (\"Length of word vector: \", len(word_to_vec_map['bat']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" ## creating tokenizer\n\ntokenizer = Tokenizer(num_words=MAX_VOCAB_SIZE)\ntokenizer.fit_on_texts(data_train.comment_text)\nword_index = tokenizer.word_index\nprint('Found %s unique tokens.' % len(word_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n## tokenizing training and testing data \n\nsequences_train = tokenizer.texts_to_sequences(data_train.comment_text)\nsequences_test = tokenizer.texts_to_sequences(data_test.comment_text)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n## creating the training data and testing data \n\n## padding the sequences to meet the maximum sequence length\ntrain_x = pad_sequences(sequences_train, maxlen=MAX_SEQ_LEN)\ntest_x = pad_sequences(sequences_test, maxlen=MAX_SEQ_LEN)\n\ntrain_y = data_train.toxic\ntest_y = data_test.toxic\n\nprint (\"Shape of training data features: \",train_x.shape)\nprint (\"Shape of training data labels: \",train_y.shape)\nprint (\"Shape of testing data features: \",test_x.shape)\nprint (\"Shape of testing data labels: \",test_y.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Creating word embeddings matrix from glove embeddings\n\n# Initializing the matrix with zeros\nembedding_matrix = np.zeros((MAX_VOCAB_SIZE, EMBEDDING_SIZE))\n\nfor word, i in tqdm(tokenizer.word_index.items()):\n    if i >= EMBEDDING_SIZE:\n        continue\n    vec = word_to_vec_map.get(word)\n    if vec is not None:\n        embedding_matrix[i,:]=vec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Creating embedding layer\n\nembedding_layer = Embedding(MAX_VOCAB_SIZE,\n                            EMBEDDING_SIZE,\n                            embeddings_initializer=Constant(embedding_matrix),\n                            input_length=MAX_SEQ_LEN,\n                            trainable=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Defining different architectures of RNN\n\n# RNN with simple unit\ndef model_SimpleRNN(input_shape):\n    input_word_ids = Input(shape=(input_shape,), dtype='int32', name=\"input_word_ids\")\n    embedded_sequences = embedding_layer(input_word_ids)\n    lstm_output = SimpleRNN(128,return_sequences=False)(embedded_sequences)\n    out = Dense(1, activation='sigmoid')(lstm_output)\n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[AUC()])\n    return model\n\n# RNN with LSTM unit\ndef model_LSTM(input_shape):\n    input_word_ids = Input(shape=(input_shape,), dtype='int32', name=\"input_word_ids\")\n    embedded_sequences = embedding_layer(input_word_ids)\n    lstm_output = LSTM(128,return_sequences=False)(embedded_sequences)\n    out = Dense(1, activation='sigmoid')(lstm_output)\n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[AUC()])\n    return model\n\n\n# RNN with 2 layers of LSTM units\ndef model_LSTM_deep(input_shape):\n    input_word_ids = Input(shape=(input_shape,), dtype='int32', name=\"input_word_ids\")\n    embedded_sequences = embedding_layer(input_word_ids)\n    lstm_output = LSTM(128,return_sequences=True)(embedded_sequences)\n    lstm_output2 = LSTM(128,return_sequences=False)(lstm_output)\n    out = Dense(1, activation='sigmoid')(lstm_output2)\n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[AUC()])\n    return model\n\n# RNN with 2 layers of bi-directional LSTM units\ndef model_LSTM_deep_bi(input_shape):\n    input_word_ids = Input(shape=(input_shape,), dtype='int32', name=\"input_word_ids\")\n    embedded_sequences = embedding_layer(input_word_ids)\n    lstm_output = Bidirectional(LSTM(128,return_sequences=True))(embedded_sequences)\n    lstm_output2 = LSTM(128,return_sequences=False)(lstm_output)\n    out = Dense(1, activation='sigmoid')(lstm_output2)\n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[AUC()])\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Simple RNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initializing the model\n\nmodel_simple = model_SimpleRNN(MAX_SEQ_LEN)\nmodel_simple.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Training the model\n\nmodel_simple.fit(train_x, train_y,\n          batch_size=128,\n          epochs=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_simple = model_simple.predict(test_x, batch_size=128)\nauc_simple = roc_auc_score(test_y, predictions_simple)\nprint (\"AUC for simple RNN unit: \",auc_simple)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LSTM"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initializing the model\n\nmodel_lstm = model_LSTM(MAX_SEQ_LEN)\nmodel_lstm.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Training the model\n\nmodel_lstm.fit(train_x, train_y,\n          batch_size=128,\n          epochs=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_lstm = model_lstm.predict(test_x, batch_size=128)\nauc_lstm = roc_auc_score(test_y, predictions_lstm)\nprint (\"AUC for LSTM unit: \",auc_lstm)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2 Layers of LSTM"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initializing the model\n\nmodel_lstm_deep = model_LSTM_deep(MAX_SEQ_LEN)\nmodel_lstm_deep.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Training the model\n\nmodel_lstm_deep.fit(train_x, train_y,\n          batch_size=128,\n          epochs=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_lstm_deep = model_lstm_deep.predict(test_x, batch_size=128)\nauc_lstm_deep = roc_auc_score(test_y, predictions_lstm_deep)\nprint (\"AUC for 2 Layers LSTM units: \",auc_lstm_deep)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## 2 Layers of Bi-directional LSTM"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initializing the model\n\nmodel_lstm_deep_bi = model_LSTM_deep_bi(MAX_SEQ_LEN)\nmodel_lstm_deep_bi.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\n# Training the model\n\nmodel_lstm_deep_bi.fit(train_x, train_y,\n          batch_size=128,\n          epochs=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions_lstm_deep_bi = model_lstm_deep_bi.predict(test_x, batch_size=128)\nauc_lstm_deep_bi = roc_auc_score(test_y, predictions_lstm_deep_bi)\nprint (\"AUC for 2 Layers Bi-directional LSTM units: \",auc_lstm_deep_bi)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}