{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Text processing using Keras\nIn this kernel we will explore different modles that we can use for text processing.\n\n## Content\n1. [Data Preparation](#1) \n2. [Basic Model](#2)\n3. [First Keras Model](#3)\n4. [Word Embeddings](#4)\n5. [Keras Embedding Layer](#5)\n6. [Using Pretrained Word Embeddings](#6)\n   1. [GloVe](#6-1)\n   2. [Wiki](#6-2)\n   3. [Word2Vec](#6-3)\n7. [CONVNET](#7)\n8. [CUDNNLSTM](#8)\n9. [CUDNNGRU](#9)\n10. [Create your model](#10)\n11. [References](#11)\n\nTo be continued ..."},{"metadata":{},"cell_type":"markdown","source":"<a id=\"1\"></a>\n# Data preparation"},{"metadata":{},"cell_type":"markdown","source":"## Import the necessary libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n%matplotlib inline\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom keras.models import Sequential\nfrom keras import layers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b567a4e6d95fcbca911bcc8324ce9bec75333b46"},"cell_type":"code","source":"!ls ../input","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5a305b0523cd2cadcb213a41fe09fdfab78c4a3"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let have a look inside train data:"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df.info()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The question column contains each sample text."},{"metadata":{"trusted":true,"_uuid":"2f1d25a36727e2f6a0362fb7338b543819cad5c7"},"cell_type":"code","source":"train_df.loc[1:3][\"question_text\"]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false},"cell_type":"markdown","source":"First we fill null entries from train and test data set."},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = train_df[\"question_text\"].fillna(\"kh\").values\nX_test = test_df[\"question_text\"].fillna(\"kh\").values\ny = train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For ploting performance of models we need this plot function."},{"metadata":{"trusted":true,"_uuid":"ceea2fcadbcd898c2e3fa4544e0369312ea2281a"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nplt.style.use('ggplot')\n\ndef plot_history(history):\n    acc = history.history['acc']\n    val_acc = history.history['val_acc']\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n    x = range(1, len(acc) + 1)\n\n    plt.figure(figsize=(12, 5))\n    plt.subplot(1, 2, 1)\n    plt.plot(x, acc, 'b', label='Training acc')\n    plt.plot(x, val_acc, 'r', label='Validation acc')\n    plt.title('Training and validation accuracy')\n    plt.legend()\n    plt.subplot(1, 2, 2)\n    plt.plot(x, loss, 'b', label='Training loss')\n    plt.plot(x, val_loss, 'r', label='Validation loss')\n    plt.title('Training and validation loss')\n    plt.legend()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40b41585a22b6fedbd8aa65bebc185d5d04cf7ce"},"cell_type":"markdown","source":"<a id=\"2\"></a>\n# Basic model: Logistic Regression "},{"metadata":{"trusted":true,"_uuid":"c95b2c7ba70a9c4c532d5458f9922c1437083fb7"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer = CountVectorizer()\nvectorizer.fit(list(X_train))\nX_train_vec = vectorizer.transform(list(X_train))\nX_test_vec  = vectorizer.transform(list(X_test))\n\n#feature selection\nfrom sklearn.feature_selection import SelectKBest, chi2\nmax_features = 50000\nch2 = SelectKBest(chi2, max_features)\nx_train = ch2.fit_transform(X_train_vec, y)\nx_test = ch2.transform(X_test_vec)\n\nfrom sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\nfrom sklearn.linear_model import LogisticRegression\nclassifier = LogisticRegression()\nclassifier.fit(x_train, y)\npre = classifier.predict_proba(x_test)\n\n# For submission \n#y_pre= [ np.argmax(i) for i in pre]\n#submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_pre})\n#submit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"848aa4b88ec308fe111dc3e0ca9ec2d232a95e03"},"cell_type":"markdown","source":"<a id=\"3\"></a>\n# First Keras Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\n\nvectorizer = TfidfVectorizer()\nX_train_vec= vectorizer.fit_transform(list(X_train))\nX_test_vec = vectorizer.transform(list(X_test))\n\nfeature_names = vectorizer.get_feature_names()\nlen(feature_names)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#feature selection\nmax_features = 50000\nch2 = SelectKBest(chi2, max_features)\nx_train = ch2.fit_transform(X_train_vec, y)\nx_test = ch2.transform(X_test_vec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0b816605c5d75cac9ba2aeb91f830fc08b0676d"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras import layers\n\nmodel = Sequential()\nmodel.add(layers.Dense(10, input_dim=max_features, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))\n\nmodel.compile(loss='binary_crossentropy',\n             optimizer='adam',\n             metrics=['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\n#vars\nbatch_size = 32\nepochs = 4\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\n\ny_pred = model.predict(x_test, batch_size=1024)\n\nplot_history(hist)\n\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\n\n#for submission\n#submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\n#submit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c98166a41f040a6f0be58f13ba2e949c27578a8d"},"cell_type":"code","source":"# Performance \nloss, accuracy = model.evaluate(X_tra,y_tra, verbose=False)\nprint(\"Training split Accuracy: {:.4f}\".format(accuracy))\nloss, accuracy = model.evaluate(X_val,y_val, verbose=False)\nprint(\"Validation Accuracy:  {:.4f}\".format(accuracy))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a49ef4f6d76aa0532dfe1bcc4d2f3fa22a8a5a64"},"cell_type":"markdown","source":"<a id=\"4\"></a>\n# Word Embeddings"},{"metadata":{"_uuid":"ff9c0389de2ec0b0122a0c79dc751a15d30de845"},"cell_type":"markdown","source":"\"The word embeddings do not understand the text as a human would, but they rather map the statistical structure of the language used in the corpus. Their aim is to map semantic meaning into a geometric space. This geometric space is then called the embedding space. vector arithmetic should become possible. A famous example in this field of study is the ability to map King - Man + Woman = Queen.\"\n\nHow to generate that:\n1. One way is to train your word embeddings during the training of your neural network. \n2. The other way is by using pretrained word embeddings which you can directly use in your model. There you have the option to either leave these word embeddings unchanged during training or you train them also."},{"metadata":{"trusted":true,"_uuid":"9bd76db8911183537b406f2669398bdff3f08a46"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\ntokenizer = Tokenizer(num_words=5000)\ntokenizer.fit_on_texts(list(X_train))\n\nx_train = tokenizer.texts_to_sequences(list(X_train))\nx_test = tokenizer.texts_to_sequences(list(X_test))\nvocab_size = len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28e34b6cdfddf42ca42fc2ac1d2f4fd71c90991b"},"cell_type":"code","source":"print(list(X_train)[2])\nx_train[2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7f349490734c86b6b67bd7a56a3f366785d836d"},"cell_type":"code","source":"for word in [ 'why', 'does', 'velocity', 'affect', 'time']:\n    print('{}: {}'.format(word, tokenizer.word_index[word]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c969a23ed969db3f25b4fe6753c88eecd1bcf59"},"cell_type":"code","source":"# make all tokenized sentences in same size\nfrom keras.preprocessing.sequence import pad_sequences\nmaxlen = 100\nx_train = pad_sequences(x_train, padding='post', maxlen=maxlen)\nx_test = pad_sequences(x_test, padding='post', maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f767d57bb9426825dad5645b927b5fe83d64c97"},"cell_type":"code","source":"print(list(X_train)[2])\nprint(x_train[2])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca504610bce5abbd028b79c10f0669cc286fd83d"},"cell_type":"code","source":"print(x_train.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9d1448c7d5def3edc8239dfbf257fc81f9679d4c"},"cell_type":"markdown","source":"<a id=\"5\"></a>\n# Keras Embedding Layer\nIn this state we tokenized data and they are just hardcoded. We will learn new embedding space using keras embedding layer that takes the previously calculated integers and maps them to a dense vector of the embedding."},{"metadata":{"_uuid":"b9c7af9ae631cf9f9154db4638a48300976bac52"},"cell_type":"markdown","source":"One way of using keras Embedding layer would be to take the output of the embedding layer and fed it into a Dense layer. In order to do this you have to add a Flatten layer in between that prepares the sequential input for the Dense layer:"},{"metadata":{"trusted":true,"_uuid":"223e88fe0c6d00b34c63c38532901887d6b9068b"},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras import layers\n\nembedding_dim = 50\n\nmodel = Sequential()\nmodel.add(layers.Embedding(input_dim=vocab_size, \n                           output_dim=embedding_dim, \n                           input_length=maxlen))\nmodel.add(layers.Flatten())\nmodel.add(layers.Dense(10, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\nmodel.summary()\n\nbatch_size = 32\nepochs = 4\n\nfrom sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\n\ny_pred = model.predict(x_test, batch_size=1024)\nplot_history(hist)\n\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\n#for submission \n#submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\n#submit_df.to_csv(\"submission.csv\", index=False)\n\n#performance\nloss, accuracy = model.evaluate(X_tra,y_tra, verbose=False)\nprint(\"Training split Accuracy: {:.4f}\".format(accuracy))\nloss, accuracy = model.evaluate(X_val,y_val, verbose=False)\nprint(\"Validation Accuracy:  {:.4f}\".format(accuracy))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"083c361fa9c05df0c82703e1a8114c50f235de4f"},"cell_type":"markdown","source":"The other way is the usage of pooling layer. "},{"metadata":{"_uuid":"e5c591bc8c477ab27d6a52e37bebdbec08a06ec6","trusted":true},"cell_type":"code","source":"from keras.models import Sequential\nfrom keras import layers\n\nembedding_dim = 50\n\nmodel = Sequential()\nmodel.add(layers.Embedding(input_dim=vocab_size, \n                           output_dim=embedding_dim, \n                           input_length=maxlen))\nmodel.add(layers.GlobalMaxPool1D())\nmodel.add(layers.Dense(10, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\nmodel.summary()\n\nbatch_size = 32\nepochs = 5\n\nfrom sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\n\ny_pred = model.predict(x_test, batch_size=1024)\nplot_history(hist)\n\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\n#submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\n#submit_df.to_csv(\"submission.csv\", index=False)\n\nloss, accuracy = model.evaluate(X_tra,y_tra, verbose=False)\nprint(\"Training split Accuracy: {:.4f}\".format(accuracy))\nloss, accuracy = model.evaluate(X_val,y_val, verbose=False)\nprint(\"Validation Accuracy:  {:.4f}\".format(accuracy))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75951e00a2a2c9cffcb5ea8ba2a180463414d38e"},"cell_type":"markdown","source":"<a id=\"6\"></a>\n# Using Pretrained Word Embeddings"},{"metadata":{"_uuid":"e737cfc11fcaae194be199b944a3ed2068334ea7"},"cell_type":"markdown","source":"Instead of training embedded space, we can use a precomputed embedding space that trained on a much larger corpus. There are different approaches for generating embedding space:\n\n1. The most popular methods are Word2Vec developed by Google; Word2Vec employes neural networks for training space.\n2. GloVe (Global Vectors for Word Representation) developed by the Stanford NLP Group; GloVe achieves this with a co-occurrence matrix and by using matrix factorization.\n\nActually both of them applying dimensionality reduction techniquies, where Word2Vec is more accurate and GloVe is faster to compute.\nFor this experiment, inside ../input/embeddings/ directory there are different embedding space which are trained on different corpora:"},{"metadata":{"trusted":true,"_uuid":"0f39262ea667a0c9a3a880e741ff9ad3f4ea615d"},"cell_type":"code","source":"! ls ../input/embeddings/*/","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" ## Load the embedding matrix "},{"metadata":{"_uuid":"f25de6ff417bd9fda9b5afdd338f607f3b5e4f0e"},"cell_type":"markdown","source":"The following function help us to generate embedding matrix that will be load in our model. Each line in the file starts with the word and is followed by the embedding vector for the particular word. We don’t need all words, we just focus on only the words that we have in our vocabulary."},{"metadata":{"trusted":true,"_uuid":"d8065a88e7899121ff93dafc3769db6d787d2865"},"cell_type":"code","source":"def create_embedding_matrix(filepath, word_index, embedding_dim):\n    vocab_size = len(word_index) + 1  # Adding again 1 because of reserved 0 index\n    embedding_matrix = np.zeros((vocab_size, embedding_dim))\n\n    with open(filepath) as f:\n        for line in f:\n            word, *vector = line.split(' ')\n            if word in word_index:\n                idx = word_index[word] \n                embedding_matrix[idx] = np.array(vector, dtype=np.float32)[:embedding_dim]\n\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<a id=\"6-1\"></a>\n## GloVe"},{"metadata":{"trusted":true,"_uuid":"1d643cf4f26fa46055d84cd391bf5cfa264e1ac7"},"cell_type":"code","source":"embedding_dim = 50\nembedding_matrix_glove = create_embedding_matrix('../input/embeddings/glove.840B.300d/glove.840B.300d.txt',\n        tokenizer.word_index, embedding_dim)\n\n#embedding_matrix.shape\n\nnonzero_elements = np.count_nonzero(np.count_nonzero(embedding_matrix_glove, axis=1))\nnonzero_elements / vocab_size","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This means 0.55% of the vocabulary is covered by the pretrained model, let check coverage of other pretrained models."},{"metadata":{},"cell_type":"markdown","source":"<a id=\"6-2\"></a>\n## Wiki"},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix_wiki = create_embedding_matrix('../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec',\n        tokenizer.word_index, embedding_dim)\n\nnonzero_elements = np.count_nonzero(np.count_nonzero(embedding_matrix_wiki, axis=1))\nnonzero_elements / vocab_size","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"<a id=\"6-3\"></a>\n"},{"metadata":{},"cell_type":"markdown","source":"As you see Glove pretrained model has better vocab coverage than wiki."},{"metadata":{},"cell_type":"markdown","source":"<a id=\"6-3\"></a>\n## Word2Vec"},{"metadata":{},"cell_type":"markdown","source":"For this corpus we modified create embedding matrix to be compatible with google corpora,"},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors\ndef create_embedding_matrix_google(filepath, word_index, embedding_dim):\n    vocab_size = len(word_index) + 1  # Adding again 1 because of reserved 0 index\n    embedding_matrix = np.zeros((vocab_size, embedding_dim))\n\n    wv_from_bin = KeyedVectors.load_word2vec_format(filepath, binary=True) \n    for word, vector in zip(wv_from_bin.vocab, wv_from_bin.vectors):\n        if word in word_index:\n                idx = word_index[word] \n                embedding_matrix[idx] = np.array(vector, dtype=np.float32)[:embedding_dim]\n\n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"610927440076a2a6f4be5b7b3f8399eb70c5aaa7"},"cell_type":"code","source":"filepath = \"../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\"\n\nembedding_matrix_google = create_embedding_matrix_google(filepath,tokenizer.word_index, embedding_dim)\n#embedding_matrix_google.shape\nnonzero_elements = np.count_nonzero(np.count_nonzero(embedding_matrix_google, axis=1))\nnonzero_elements / vocab_size","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"So we decide to use the first pretained model which is done by GloVe. Now we can use the word embeddings in our models. In the first model we use "},{"metadata":{"trusted":true,"_uuid":"c36b284dc496046cacb9df1a1a843617366872b1"},"cell_type":"code","source":"model = Sequential()\nmodel.add(layers.Embedding(vocab_size, embedding_dim, \n                           weights=[embedding_matrix_glove], \n                           input_length=maxlen, \n                           trainable=False))\nmodel.add(layers.GlobalMaxPool1D())\nmodel.add(layers.Dense(10, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\nmodel.summary()\n\nbatch_size = 32\nepochs = 5\n\nfrom sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\n\ny_pred = model.predict(x_test, batch_size=1024)\nplot_history(hist)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"In the previous model the word embeddings are not additionally trained, now we will check model performs if we allow the embedding to be trained by using trainable=True:"},{"metadata":{"trusted":true,"_uuid":"8e34f1d0353d06d12e848da5b25b1900eac9cb76"},"cell_type":"code","source":"model = Sequential()\nmodel.add(layers.Embedding(vocab_size, embedding_dim, \n                           weights=[embedding_matrix_glove], \n                           input_length=maxlen, \n                           trainable=True))\nmodel.add(layers.GlobalMaxPool1D())\nmodel.add(layers.Dense(10, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\nmodel.summary()\n\nbatch_size = 32\nepochs = 5\n\nfrom sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\n\ny_pred = model.predict(x_test, batch_size=1024)\nplot_history(hist)\n\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\n#submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\n#submit_df.to_csv(\"submission.csv\", index=False)\n\nloss, accuracy = model.evaluate(X_tra,y_tra, verbose=False)\nprint(\"Training split Accuracy: {:.4f}\".format(accuracy))\nloss, accuracy = model.evaluate(X_val,y_val, verbose=False)\nprint(\"Validation Accuracy:  {:.4f}\".format(accuracy))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9775c02b6a22cb2e2860e42571b78102889f12ea"},"cell_type":"markdown","source":"<a id=\"7\"></a>\n# CONVNET"},{"metadata":{"trusted":true,"_uuid":"b270cc93bafecbc727d2df2957039cba569b00dc"},"cell_type":"code","source":"embedding_dim = 100\n\nmodel = Sequential()\nmodel.add(layers.Embedding(vocab_size, embedding_dim, input_length=maxlen))\nmodel.add(layers.Conv1D(128, 5, activation='relu'))\nmodel.add(layers.GlobalMaxPooling1D())\nmodel.add(layers.Dense(10, activation='relu'))\nmodel.add(layers.Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\nmodel.summary()\n\nbatch_size = 32\nepochs = 5\n\nfrom sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\n\ny_pred = model.predict(x_test, batch_size=1024)\nplot_history(hist)\n\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\n#submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\n#submit_df.to_csv(\"submission.csv\", index=False)\n\nloss, accuracy = model.evaluate(X_tra,y_tra, verbose=False)\nprint(\"Training split Accuracy: {:.4f}\".format(accuracy))\nloss, accuracy = model.evaluate(X_val,y_val, verbose=False)\nprint(\"Validation Accuracy:  {:.4f}\".format(accuracy))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b81aea162347eff730e8799dee5fe0056777df47"},"cell_type":"markdown","source":"<a id=\"8\"></a>\n# CuDNNLSTM"},{"metadata":{"trusted":true,"_uuid":"0850f9fa426c942e731ea566a15c58e85c8c5ffc"},"cell_type":"code","source":"from keras.layers import LSTM, Dense, Bidirectional, Input,Dropout,BatchNormalization, CuDNNGRU, CuDNNLSTM\n\nembedding_dim = 100\nbatch_size = 32\nepochs = 1\n\nmodel = Sequential()\nmodel.add(layers.Embedding(vocab_size, embedding_dim, input_length=maxlen))\nmodel.add(CuDNNLSTM(128,return_sequences=True))\nmodel.add(layers.GlobalMaxPooling1D())\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\n\nmodel.summary()\n\nfrom sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\n\ny_pred = model.predict(x_test, batch_size=1024)\n\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\n#submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\n#submit_df.to_csv(\"submission.csv\", index=False)\n\nloss, accuracy = model.evaluate(X_tra,y_tra, verbose=False)\nprint(\"Training split Accuracy: {:.4f}\".format(accuracy))\nloss, accuracy = model.evaluate(X_val,y_val, verbose=False)\nprint(\"Validation Accuracy:  {:.4f}\".format(accuracy))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7bea4673fa37ae238dcbe48a8b19597a0d57b717"},"cell_type":"markdown","source":"<a id=\"9\"></a>\n# CuDNNGRU"},{"metadata":{"trusted":true,"_uuid":"62f778170af4cfecf4a5e28d0717d2192d683c41"},"cell_type":"code","source":"from keras.layers import LSTM, Dense, Bidirectional, Input,Dropout,BatchNormalization, CuDNNGRU, CuDNNLSTM\n\nembedding_dim = 100\nbatch_size = 32\nepochs = 1\n\nmodel = Sequential()\nmodel.add(layers.Embedding(vocab_size, embedding_dim, input_length=maxlen))\nmodel.add(CuDNNGRU(64, return_sequences=True))\nmodel.add(layers.GlobalMaxPooling1D())\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam',\n              loss='binary_crossentropy',\n              metrics=['accuracy'])\nmodel.summary()\n\nfrom sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\n\ny_pred = model.predict(x_test, batch_size=1024)\n\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\n#submit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\n#submit_df.to_csv(\"submission.csv\", index=False)\n\nloss, accuracy = model.evaluate(X_tra,y_tra, verbose=False)\nprint(\"Training split Accuracy: {:.4f}\".format(accuracy))\nloss, accuracy = model.evaluate(X_val,y_val, verbose=False)\nprint(\"Validation Accuracy:  {:.4f}\".format(accuracy))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d3774e6b9e559330f35e8da6b3cb6230c8c736ab"},"cell_type":"markdown","source":"<a id=\"10\"></a>\n# Create your model"},{"metadata":{},"cell_type":"markdown","source":"This model comes from [keras starter](https://www.kaggle.com/christofhenkel/keras-starter) kernel. In this model we will define our custom model."},{"metadata":{"trusted":true,"_uuid":"d702fb1a3ae2b88745f178a2753c60f43ecd32ce"},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense, Embedding, concatenate\nfrom keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D\n\nmaxlen = 100\nmax_features = 50000\n\ndef get_model():\n    inp = Input(shape=(maxlen, ))\n    x = Embedding(max_features, 100)(inp)\n    x = CuDNNGRU(64, return_sequences=True)(x)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    conc = concatenate([avg_pool, max_pool])\n    outp = Dense(1, activation=\"sigmoid\")(conc)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n\n    return model\n\nmodel = get_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"batch_size = 32\nepochs = 1\n\nfrom sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)\n\nhist = model.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val),\n                  verbose=True)\n\ny_pred = model.predict(x_test, batch_size=1024)\n\ny_te = (y_pred[:,0] > 0.5).astype(np.int)\nsubmit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\nsubmit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false},"cell_type":"markdown","source":"<a id=\"11\"></a>\n# References \n*  https://realpython.com/python-keras-text-classification/"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}