{"cells":[{"metadata":{"_uuid":"cf64e8851351fb558ddf6bcf2d9bc3a5614dc6ef"},"cell_type":"markdown","source":"### From Version 23, which is the best vesion for both public and private dataset, This a draft notebook, a detailed tutorial with my train of thoughts about this notebook wil be ready soon.\n"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"import pandas as pd\n\nimport matplotlib\nimport numpy as np\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\n\nimport nltk","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"df = pd.read_csv(\"../input/train.csv\")\n\ndf_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17b20f13609657a4668159762c0e7d81b2d558c0","trusted":false},"cell_type":"code","source":"df.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"59ea39d66cf8a545d0596ed186ab6453906e5446","trusted":false},"cell_type":"code","source":"df.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"98ec366ffbddcc8907afd9cd0b451a84863230ef","trusted":false},"cell_type":"code","source":"df_test.head(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3d0a7d37cd150535a5a5d587cfcbf75d0205956","trusted":false},"cell_type":"code","source":"df_test.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e2a1d8254c84a2a67930f9a54b0057490c6d4d4c","trusted":false},"cell_type":"code","source":"def tokenize_the_text(phrases):\n    \n    from nltk.tokenize import word_tokenize\n    from nltk.text import Text\n    \n    tokens = [word for word in phrases]\n    tokens = [word.lower() for word in tokens]\n    tokens = [word_tokenize(word) for word in tokens]\n    \n    return tokens\n\n#crude_tokens = tokenize_the_text(df.question_text)\n#print(crude_tokens[0:10])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f920a342fa3e0298e67c53f6fd3b6bf86e6b6b19","trusted":false},"cell_type":"code","source":"def create_a_vocab(tokens):\n    \n    vocab = set()\n\n    for sentence in tokens:\n        for word in sentence:\n            vocab.add(word)\n\n    vocab = list(vocab)\n\n    return vocab\n    \n#vocab = create_a_vocab(crude_tokens)\n\nvocab = create_a_vocab(tokenize_the_text(df.question_text))\n\nprint(\"Vocabulary size:\", len(vocab), \"words\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3e9ff461ef015e09a4a5f534c56de024501ba0a7","trusted":false},"cell_type":"code","source":"def removing_stopwords(tokens_custom_cleaned):\n\n    from nltk.corpus import stopwords\n    stop_words = stopwords.words('english')\n    tokens_custom_cleaned_and_without_stopwords = []\n    for sentence in tokens_custom_cleaned:\n        tokens_custom_cleaned_and_without_stopwords.append([word for word in sentence if word not in stop_words])\n        \n    return tokens_custom_cleaned_and_without_stopwords\n\ntokens_without_stopwords = removing_stopwords(tokenize_the_text(df.question_text))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a4ebb3e12a34dfc4e5f3d1470b4384d7c46dc79","trusted":false},"cell_type":"code","source":"vocab = create_a_vocab(tokens_without_stopwords)\n\nprint(\"Vocabulary size after removing stopwords:\", len(vocab), \"words\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"565065f95e2f7c3700ab13f53d463226b54d61ce","trusted":false},"cell_type":"code","source":"def lemmatizing_the_tokens(tokens_custom_cleaned_and_without_stopwords):\n\n    from nltk.stem.wordnet import WordNetLemmatizer \n    lem = WordNetLemmatizer()\n\n    tokens_custom_cleaned_and_without_stopwords_and_lemmatized = []\n\n    for sentence in tokens_custom_cleaned_and_without_stopwords:\n        tokens_custom_cleaned_and_without_stopwords_and_lemmatized.append([lem.lemmatize(word, pos='v') for word in sentence])\n        \n    return tokens_custom_cleaned_and_without_stopwords_and_lemmatized\n\n\ntokens_without_stopwords_and_lemmatized = lemmatizing_the_tokens(tokens_without_stopwords)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d1775be8fa242ec1e55d1232b118f56c90145cf","trusted":false},"cell_type":"code","source":"vocab = create_a_vocab(tokens_without_stopwords_and_lemmatized)\n\nprint(\"Vocabulary size after removing stopwords and lemmatizing the text:\", len(vocab), \"words\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe34fb76097ee68a544d17346d8d752b19d18c16","trusted":false},"cell_type":"code","source":"### do the same and for the testSet\n\n\ntokens_without_stopwords_test = removing_stopwords(tokenize_the_text(df_test.question_text))\ntokens_without_stopwords_and_lemmatized_test = lemmatizing_the_tokens(tokens_without_stopwords_test)\n\nvocab_test = create_a_vocab(tokens_without_stopwords_and_lemmatized_test)\n\nprint(\"Vocabulary size after removing stopwords and lemmatizing the text:\", len(vocab_test), \"words\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"909387b13deca492e5b7e4583e438f4f08cd8b9c","trusted":false},"cell_type":"code","source":"del tokens_without_stopwords_test\ndel tokens_without_stopwords\ndel vocab\ndel vocab_test","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"98f4a00bb4d49b7f043d6d7e09bcab03ed56072f","trusted":false},"cell_type":"code","source":"# Keras Libraries\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Sequential\nfrom keras.layers import Dense, Flatten, LSTM, GRU, Conv1D, MaxPooling1D, Dropout, SpatialDropout1D, Bidirectional, Activation,GlobalAveragePooling1D, GlobalMaxPooling1D\nfrom keras.layers.embeddings import Embedding\nfrom sklearn.model_selection import train_test_split\nfrom keras.models import load_model\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import classification_report","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"07bf7060e60f3258db9ae6133759554973203697","trusted":false},"cell_type":"code","source":"def get_embeddings_dict():\n\n    import numpy as np\n\n    filename = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n\n    glove_w2v_embeddings_index = dict()\n    f = open(filename, \"r\", encoding='utf-8')\n    for line in f:\n        values = line.split(' ')\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype='float32')\n        glove_w2v_embeddings_index[word] = coefs\n    f.close()\n    \n    return glove_w2v_embeddings_index\n\n\nglove_w2v_embeddings_index = get_embeddings_dict()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"35d1d4a262ea2c29fca16fe15dd15cabd0638c35","trusted":false},"cell_type":"code","source":"###\n#tokens_uncleaned = tokenize_the_text(df.question_text.values)\n#tokens_uncleaned_test = tokenize_the_text(df_test.question_text.values)\n#vocab_all_corpus = create_a_vocab(tokenize_the_text(df.question_text.values) + tokenize_the_text(df_test.question_text.values))\n\n#sentences = [' '.join(sent) for sent in tokens_uncleaned]\n#sentences_test = [' '.join(sent) for sent in tokens_uncleaned_test]\n\n\nall_corpus = [' '.join(sent) for sent in tokens_without_stopwords_and_lemmatized] + [' '.join(sent) for sent in tokens_without_stopwords_and_lemmatized_test]\nmax_len = max([len(elem.split()) for elem in all_corpus])\n#print(max_len)\n###\n\n\ntokenizer = Tokenizer(lower=True, filters='')\ntokenizer.fit_on_texts(all_corpus)\n\nvocabulary_size = len(tokenizer.word_index) + 1\n#print(vocabulary_size)\n\n\n\nX = tokenizer.texts_to_sequences([' '.join(sent) for sent in tokens_without_stopwords_and_lemmatized])\ny = df.target.values\nX = pad_sequences(X, maxlen=max_len)\n\n\nxtrain, xvalid, ytrain, yvalid = train_test_split(X, y, stratify=y, random_state=42, test_size=0.3, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1b12ef8205a04a43b941599495656fb6dc39f6c","trusted":false},"cell_type":"code","source":"del X\ndel y\ndel tokens_without_stopwords_and_lemmatized\ndel all_corpus","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"20682234f5ff44ef1f2487eb0da9012c19fb0bf7","trusted":false},"cell_type":"code","source":"import textblob\n\nembedding_dim = 300\n\ndef get_embedding_matrix():\n    \n    embedding_matrix = np.zeros((vocabulary_size, embedding_dim + 2))\n\n    for word, i in tokenizer.word_index.items():\n        if i > vocabulary_size:\n            continue\n        embedding_vector = glove_w2v_embeddings_index.get(word)\n        word_sentiment = textblob.TextBlob(word).sentiment\n        if embedding_vector is not None:\n            embedding_matrix[i] = np.append(embedding_vector, [word_sentiment.polarity, word_sentiment.subjectivity])\n        else:\n            embedding_matrix[i, -2:] = [word_sentiment.polarity, word_sentiment.subjectivity]\n            \n    return embedding_matrix\n\nembedding_matrix = get_embedding_matrix()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d70cbfd77534b17df69ed1ec69ac151d59f5a64","trusted":false},"cell_type":"code","source":"del glove_w2v_embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ee94c08d7799ff853bf6f7bc5f50ebade612b4e4","trusted":false},"cell_type":"code","source":"def plot_history(history):\n    acc = history.history['categorical_accuracy']\n    val_acc = history.history['val_categorical_accuracy']\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n    x = range(1, len(acc) + 1)\n\n    plt.figure(figsize=(12, 5))\n    plt.subplot(1, 2, 1)\n    plt.plot(x, acc, 'b', label='Training acc')\n    plt.plot(x, val_acc, 'r', label='Validation acc')\n    plt.title('Training and validation accuracy')\n    plt.legend()\n    plt.subplot(1, 2, 2)\n    plt.plot(x, loss, 'b', label='Training loss')\n    plt.plot(x, val_loss, 'r', label='Validation loss')\n    plt.title('Training and validation loss')\n    plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"51e54d5cef365ec09007259ecc0526a5cbb524e7"},"cell_type":"markdown","source":"### LSTM"},{"metadata":{"_uuid":"cecd479ed42136e14f4268f4280acc44f965181e","trusted":false},"cell_type":"code","source":"def build_dl_lstm_model(xtrain, ytrain, xvalid, yvalid, num_of_epochs, filename):\n\n    ## Network architecture\n    from keras.utils import to_categorical\n    from keras.callbacks import ModelCheckpoint\n    from keras.callbacks import EarlyStopping\n    from keras.layers import Masking\n    from keras.initializers import Constant\n    import time\n    \n    import warnings\n    warnings.simplefilter(action='ignore', category=FutureWarning)\n    \n    start_time = time.time()\n    \n    \n    from numpy.random import seed\n    seed(42)\n    from tensorflow import set_random_seed\n    set_random_seed(42)\n\n    embedding_size= embedding_dim + 2\n    batch_size = 512\n    dropouts = 0.2\n    epochs = num_of_epochs\n    \n    model=Sequential()\n    model.add(Embedding(input_dim = vocabulary_size, output_dim = embedding_size, input_length = max_len, \n                        weights=[embedding_matrix], trainable = False, mask_zero=True))\n    \n    model.add(SpatialDropout1D(dropouts))\n    \n    model.add(LSTM(int((embedding_size/2)-50), recurrent_dropout=dropouts, dropout=dropouts, return_sequences=False))\n    \n    model.add(Dense(2, activation='softmax'))\n\n    print(model.summary())\n\n    model.compile(loss='categorical_crossentropy', optimizer = 'nadam', metrics=['categorical_accuracy']) # RMSprop\n\n    '''\n    saves the model weights after each epoch if the val_acc loss decreased\n    '''\n    checkpointer = ModelCheckpoint(monitor='val_categorical_accuracy', mode='max', filepath=''+filename+'.hdf5', verbose=2, save_best_only=True)\n    earlyStopping = EarlyStopping(monitor='val_categorical_accuracy', min_delta=0, patience=1, verbose=0, mode='max')\n\n    history = model.fit(x = xtrain, y = to_categorical(ytrain), validation_data=(xvalid, to_categorical(yvalid)), epochs=epochs, batch_size=batch_size, verbose=2, callbacks=[checkpointer, earlyStopping])\n\n    model = load_model(''+filename+'.hdf5')\n    \n    elapsed_time = time.time() - start_time\n    \n    return model, history, elapsed_time","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"129dab5c689c5a04aa22b6bd1138c46ebaa1c68b"},"cell_type":"markdown","source":"### GRU"},{"metadata":{"_uuid":"85b281b8bf6f62bb94785480d76d5ff3b162779f","trusted":false},"cell_type":"code","source":"def build_dl_lstm_model(xtrain, ytrain, xvalid, yvalid, num_of_epochs, filename):\n\n    ## Network architecture\n    from keras.utils import to_categorical\n    from keras.callbacks import ModelCheckpoint\n    from keras.callbacks import EarlyStopping\n    from keras.layers import Masking\n    from keras.initializers import Constant\n    import time\n    \n    import warnings\n    warnings.simplefilter(action='ignore', category=FutureWarning)\n    \n    start_time = time.time()\n    \n    \n    from numpy.random import seed\n    seed(42)\n    from tensorflow import set_random_seed\n    set_random_seed(42)\n\n    embedding_size= embedding_dim + 2\n    batch_size = 512\n    dropouts = 0.2\n    epochs = num_of_epochs\n    \n    model=Sequential()\n    model.add(Embedding(input_dim = vocabulary_size, output_dim = embedding_size, input_length = max_len, \n                        weights=[embedding_matrix], trainable = False, mask_zero=True))\n    \n    model.add(SpatialDropout1D(dropouts))\n    \n    model.add(GRU(int((embedding_size/2)-50), recurrent_dropout=dropouts, dropout=dropouts, return_sequences=False))\n    \n    model.add(Dense(2, activation='softmax'))\n\n    print(model.summary())\n\n    model.compile(loss='categorical_crossentropy', optimizer = 'nadam', metrics=['categorical_accuracy']) # RMSprop\n\n    '''\n    saves the model weights after each epoch if the val_acc loss decreased\n    '''\n    checkpointer = ModelCheckpoint(monitor='val_categorical_accuracy', mode='max', filepath=''+filename+'.hdf5', verbose=2, save_best_only=True)\n    earlyStopping = EarlyStopping(monitor='val_categorical_accuracy', min_delta=0, patience=1, verbose=0, mode='max')\n\n    history = model.fit(x = xtrain, y = to_categorical(ytrain), validation_data=(xvalid, to_categorical(yvalid)), epochs=epochs, batch_size=batch_size, verbose=2, callbacks=[checkpointer, earlyStopping])\n\n    model = load_model(''+filename+'.hdf5')\n    \n    elapsed_time = time.time() - start_time\n    \n    return model, history, elapsed_time","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e08e1fdd561e3085413051307501406ea362cd04"},"cell_type":"markdown","source":"### LSTM + CNN"},{"metadata":{"_uuid":"cfb0c742e9235322d308384f39e1fcc270d5c45b","trusted":false},"cell_type":"code","source":"def build_dl_lstm_cnn_model(xtrain, ytrain, xvalid, yvalid, num_of_epochs, filename):\n\n    ## Network architecture\n    from keras.utils import to_categorical\n    from keras.callbacks import ModelCheckpoint\n    from keras.callbacks import EarlyStopping\n    from keras.layers import Masking\n    from keras.initializers import Constant\n    import time\n    \n    import warnings\n    warnings.simplefilter(action='ignore', category=FutureWarning)\n    \n    start_time = time.time()\n    \n    \n    from numpy.random import seed\n    seed(42)\n    from tensorflow import set_random_seed\n    set_random_seed(42)\n\n    embedding_size= embedding_dim + 2\n    batch_size = 1024\n    dropouts = 0.2\n    epochs = num_of_epochs\n    \n    model=Sequential()\n    model.add(Embedding(input_dim = vocabulary_size, output_dim = embedding_size, input_length = max_len, \n                        weights=[embedding_matrix], trainable = False))\n    \n    model.add(SpatialDropout1D(dropouts))\n    \n    model.add(GRU(units = 64, recurrent_dropout=dropouts, dropout=dropouts, return_sequences=True))\n    \n    model.add(Conv1D(128, kernel_size = 1, strides = 1,  padding='valid', activation='relu'))\n    model.add(Conv1D(256, kernel_size = 3, strides = 1,  padding='valid', activation='relu'))\n    model.add(Conv1D(512, kernel_size = 5, strides = 1,  padding='valid', activation='relu'))\n    \n    model.add(GlobalMaxPooling1D())\n    model.add(Dropout(dropouts))\n    \n    \n    model.add(Dense(2, activation='softmax'))\n\n    print(model.summary())\n\n    model.compile(loss='categorical_crossentropy', optimizer = 'nadam', metrics=['categorical_accuracy']) # RMSprop\n\n    '''\n    saves the model weights after each epoch if the val_acc loss decreased\n    '''\n    checkpointer = ModelCheckpoint(monitor='val_categorical_accuracy', mode='max', filepath=''+filename+'.hdf5', verbose=2, save_best_only=True)\n    earlyStopping = EarlyStopping(monitor='val_categorical_accuracy', min_delta=0, patience=0, verbose=0, mode='max')\n\n    history = model.fit(x = xtrain, y = to_categorical(ytrain), validation_data=(xvalid, to_categorical(yvalid)), epochs=epochs, batch_size=batch_size, verbose=2, callbacks=[checkpointer, earlyStopping])\n\n    model = load_model(''+filename+'.hdf5')\n    \n    elapsed_time = time.time() - start_time\n    \n    return model, history, elapsed_time\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c9930700446c331436724ade8fcd77d9b57ce222"},"cell_type":"markdown","source":"### GRU + CNN"},{"metadata":{"_uuid":"af58c16c3c0f8811a07291048f00e5e8adedda7d","trusted":false},"cell_type":"code","source":"def build_dl_gru_cnn_model(xtrain, ytrain, xvalid, yvalid, num_of_epochs, filename):\n\n    ## Network architecture\n    from keras.utils import to_categorical\n    from keras.callbacks import ModelCheckpoint\n    from keras.callbacks import EarlyStopping\n    from keras.layers import Masking\n    from keras.initializers import Constant\n    import time\n    \n    import warnings\n    warnings.simplefilter(action='ignore', category=FutureWarning)\n    \n    start_time = time.time()\n    \n    \n    from numpy.random import seed\n    seed(42)\n    from tensorflow import set_random_seed\n    set_random_seed(42)\n\n    embedding_size= embedding_dim + 2\n    batch_size = 1024\n    dropouts = 0.2\n    epochs = num_of_epochs\n    \n    model=Sequential()\n    model.add(Embedding(input_dim = vocabulary_size, output_dim = embedding_size, input_length = max_len, \n                        weights=[embedding_matrix], trainable = False))\n    \n    model.add(SpatialDropout1D(dropouts))\n    \n    model.add(GRU(units = 84, recurrent_dropout=dropouts, dropout=dropouts, return_sequences=True))\n    \n    model.add(Conv1D(42, kernel_size = 1, strides = 1,  padding='valid', activation='relu'))\n    #model.add(Conv1D(256, kernel_size = 3, strides = 1,  padding='valid', activation='relu'))\n    #model.add(Conv1D(512, kernel_size = 5, strides = 1,  padding='valid', activation='relu'))\n    \n    model.add(GlobalMaxPooling1D())\n    model.add(Dropout(dropouts))\n    \n    model.add(Dense(26, activation=\"relu\"))\n    model.add(Dropout(dropouts))\n        \n    model.add(Dense(2, activation='softmax'))\n\n    print(model.summary())\n\n    model.compile(loss='categorical_crossentropy', optimizer = 'nadam', metrics=['categorical_accuracy']) # RMSprop\n\n    '''\n    saves the model weights after each epoch if the val_acc loss decreased\n    '''\n    checkpointer = ModelCheckpoint(monitor='val_categorical_accuracy', mode='max', filepath=''+filename+'.hdf5', verbose=2, save_best_only=True)\n    earlyStopping = EarlyStopping(monitor='val_categorical_accuracy', min_delta=0, patience=0, verbose=0, mode='max')\n\n    history = model.fit(x = xtrain, y = to_categorical(ytrain), validation_data=(xvalid, to_categorical(yvalid)), epochs=epochs, batch_size=batch_size, verbose=2, callbacks=[checkpointer, earlyStopping])\n\n    model = load_model(''+filename+'.hdf5')\n    \n    elapsed_time = time.time() - start_time\n    \n    return model, history, elapsed_time\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6eb757aab246bf38b7ca879aabec5f38dcf77282"},"cell_type":"markdown","source":"### Bidirectional GRU + CNN"},{"metadata":{"_uuid":"39defd0e9ebc395b4fd2f4d318b0ea2b69e81d82","trusted":false},"cell_type":"code","source":"def build_dl_bidirectional_gru_cnn_model(xtrain, ytrain, xvalid, yvalid, num_of_epochs, filename):\n\n    ## Network architecture\n    from keras.utils import to_categorical\n    from keras.callbacks import ModelCheckpoint\n    from keras.callbacks import EarlyStopping\n    from keras.layers import Masking\n    from keras.initializers import Constant\n    import time\n    \n    import warnings\n    warnings.simplefilter(action='ignore', category=FutureWarning)\n    \n    start_time = time.time()\n    \n    \n    from numpy.random import seed\n    seed(42)\n    from tensorflow import set_random_seed\n    set_random_seed(42)\n\n    embedding_size= embedding_dim + 2\n    batch_size = 1024\n    dropouts = 0.2\n    epochs = num_of_epochs\n    \n    model=Sequential()\n    model.add(Embedding(input_dim = vocabulary_size, output_dim = embedding_size, input_length = max_len, \n                        weights=[embedding_matrix], trainable = False))\n    \n    model.add(SpatialDropout1D(dropouts))\n    \n    model.add(Bidirectional(GRU(units = 64, recurrent_dropout=dropouts, dropout=dropouts, return_sequences=True)))\n    \n    model.add(Conv1D(128, kernel_size = 1, strides = 1,  padding='valid', activation='relu'))\n    model.add(Conv1D(256, kernel_size = 3, strides = 1,  padding='valid', activation='relu'))\n    model.add(Conv1D(512, kernel_size = 5, strides = 1,  padding='valid', activation='relu'))\n    \n    model.add(GlobalMaxPooling1D())\n    model.add(Dropout(dropouts))\n    \n    model.add(Dense(16, activation=\"relu\"))\n    model.add(Dropout(dropouts))\n    \n    \n    model.add(Dense(2, activation='softmax'))\n\n    print(model.summary())\n\n    model.compile(loss='categorical_crossentropy', optimizer = 'nadam', metrics=['categorical_accuracy']) # RMSprop\n\n    '''\n    saves the model weights after each epoch if the val_acc loss decreased\n    '''\n    checkpointer = ModelCheckpoint(monitor='val_categorical_accuracy', mode='max', filepath=''+filename+'.hdf5', verbose=2, save_best_only=True)\n    earlyStopping = EarlyStopping(monitor='val_categorical_accuracy', min_delta=0, patience=0, verbose=0, mode='max')\n\n    history = model.fit(x = xtrain, y = to_categorical(ytrain), validation_data=(xvalid, to_categorical(yvalid)), epochs=epochs, batch_size=batch_size, verbose=2, callbacks=[checkpointer, earlyStopping])\n\n    model = load_model(''+filename+'.hdf5')\n    \n    elapsed_time = time.time() - start_time\n    \n    return model, history, elapsed_time\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aec38f004fb92d06bde4ba7a7962853ac7f5bb95","trusted":false},"cell_type":"code","source":"'''\ndl_lstm_model, history_lstm, elapsed_time = build_dl_lstm_model(xtrain, ytrain, xvalid, yvalid, num_of_epochs=30, filename=\"lstm\")\nprint(\"Elapsed time in seconds:\", elapsed_time)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"24f83d88588f8912195c87be81f33284f174655b","trusted":false},"cell_type":"code","source":"'''\ndl_lstm_cnn_model, history_lstm_cnn, elapsed_time = build_dl_lstm_cnn_model(xtrain, ytrain, xvalid, yvalid, num_of_epochs=30, filename=\"lstm_cnn\")\nprint(\"Elapsed time in seconds:\", elapsed_time)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63f16960b78dae2dd63cd31c332006467e294395","trusted":false},"cell_type":"code","source":"\ndl_gru_cnn_model, history_gru_cnn, elapsed_time = build_dl_gru_cnn_model(xtrain, ytrain, xvalid, yvalid, num_of_epochs=30, filename=\"gru_cnn\")\nprint(\"Elapsed time in seconds:\", elapsed_time)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c135431e8a80542cb8f72f6f538a785596b62742","trusted":false},"cell_type":"code","source":"'''\ndl_bidirectional_gru_cnn_model, history_bidirectional_gru_cnn, elapsed_time = build_dl_bidirectional_gru_cnn_model(xtrain, ytrain, xvalid, yvalid, num_of_epochs=30, filename=\"bidirectional_gru_cnn\")\nprint(\"Elapsed time in seconds:\", elapsed_time)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b99ed0fa007f58d1dc2bb378a5eb97f73037ffe","trusted":false},"cell_type":"code","source":"'''\nplot_history(history_lstm)\n\n\ny_pred_lstm = dl_lstm_model.predict_classes(xvalid, verbose=1, batch_size = 256)\nprint()\n\nprint(classification_report(yvalid, y_pred_lstm))\n\nprint()\nprint(\"accuracy_score\", accuracy_score(yvalid, y_pred_lstm))\n\nprint()\nprint(\"Weighted Averaged validation metrics\")\nprint(\"precision_score\", precision_score(yvalid, y_pred_lstm, average='weighted'))\nprint(\"recall_score\", recall_score(yvalid, y_pred_lstm, average='weighted'))\nprint(\"f1_score\", f1_score(yvalid, y_pred_lstm, average='weighted'))\n\nprint()\nfrom sklearn.metrics import confusion_matrix\nimport scikitplot as skplt\nsns.set(rc={'figure.figsize':(8,8)})\nskplt.metrics.plot_confusion_matrix(yvalid, y_pred_lstm)\n\n\nprint(\"elapsed time:\", round(elapsed_time), \"seconds\")\n'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"841cf212e242b77c07aab290cb7c5795a7c37fca","trusted":false},"cell_type":"code","source":"#plot_history(history_lstm_cnn)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a9ab552c6d0647540a7153a18445059ed7268f55","trusted":false},"cell_type":"code","source":"plot_history(dl_gru_cnn_model)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"38661b480682ed2f55d95c171c23ad775bd3cb18","trusted":false},"cell_type":"code","source":"xtest = tokenizer.texts_to_sequences([' '.join(sent) for sent in tokens_without_stopwords_and_lemmatized_test])\nxtest = pad_sequences(xtest, maxlen=max_len)\n\n'''\ny_pred_test_lstm = dl_lstm_model.predict_classes(xtest, verbose=1, batch_size = 512)\nsubmission = pd.DataFrame()\nsubmission['qid'] = df_test.qid\nsubmission['prediction'] = y_pred_test_lstm\n#submission['Sentiment'] = submission.Sentiment.astype(int)\nsubmission.to_csv('submission.csv',index=False)\n'''\n\n'''\ny_pred_test_lstm_cnn = dl_lstm_cnn_model.predict_classes(xtest, verbose=1, batch_size = 1024)\nsubmission = pd.DataFrame()\nsubmission['qid'] = df_test.qid\nsubmission['prediction'] = y_pred_test_lstm_cnn\n#submission['Sentiment'] = submission.Sentiment.astype(int)\nsubmission.to_csv('submission.csv',index=False)\n'''\n\n\ny_pred_test_gru_cnn = dl_gru_cnn_model.predict_classes(xtest, verbose=1, batch_size = 1024)\nsubmission = pd.DataFrame()\nsubmission['qid'] = df_test.qid\nsubmission['prediction'] = y_pred_test_gru_cnn\n#submission['Sentiment'] = submission.Sentiment.astype(int)\nsubmission.to_csv('submission.csv',index=False)\n\n'''\ny_pred_test_bidirectional_gru_cnn = dl_bidirectional_gru_cnn_model.predict_classes(xtest, verbose=1, batch_size = 1024)\nsubmission = pd.DataFrame()\nsubmission['qid'] = df_test.qid\nsubmission['prediction'] = y_pred_test_bidirectional_gru_cnn\n#submission['Sentiment'] = submission.Sentiment.astype(int)\nsubmission.to_csv('submission.csv',index=False)\n'''","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ff3895b026550466f74fb4a3e3466df11c312d2","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}