{"cells":[{"metadata":{"_uuid":"01f361ddc47e0b386595316fe3d7f4dabbd260db"},"cell_type":"markdown","source":"# The problem\n* Classify Quora questions into sincere and insincere (label- '1') \n* Can be thought of as a sequence prediction problem which can be modeled using an LSTM(or cuDNNLSTM)\n* With a bit of help from [here](https://www.kaggle.com/artgor/eda-and-lstm-cnn) and [here](https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings) I hope to create a baseline model using Glove embeddings. \n* Here is a [great](https://cran.r-project.org/web/packages/text2vec/vignettes/glove.html) overview of the GloVe model"},{"metadata":{"trusted":true,"_uuid":"82381cc7db27849d50c33b1d3e06c1e40e9766cc"},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nimport nltk\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nimport matplotlib.pyplot as plt\n\n%matplotlib inline\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, CuDNNLSTM, BatchNormalization\nfrom keras.layers import Bidirectional, GlobalMaxPool2D, Conv2D, GlobalMaxPooling1D, Conv1D, MaxPool1D,MaxPooling1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d0e37d6669ddf726426cfae8003dbc10378d9e5d"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c8ffc5677f5b1d675c97c65c6db457dc2d3af13"},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\ntrain_X = pad_sequences(train_X, maxlen=maxlen, padding='pre')\ntest_X = pad_sequences(test_X, maxlen=maxlen, padding='pre')\ntrain_y = train_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a287a6c5fc539b66c3d2b21bc3b16f307ae4102b"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n#embeddings contains the Word to vector map","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e646523529400b20af0df1792832ee59408a5d44"},"cell_type":"code","source":"#we need to still take care of words that are present in the data but not in the embeddings.\n#dict.get() method comes in very handy to avoid \"no key found errors\"\nall_embs = np.stack(embeddings_index.values()) #capture embedding statistics \nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words+1, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0aa5d8df5522c3a4f05fd4b058aa34e86ec5d79"},"cell_type":"code","source":"#defining embedding layer\ndef pretrained_embedding_layer(embedding_matrix):\n\n    vocab_len = max_features+1 #keras requirement\n    embedding_layer = Embedding(vocab_len, embed_size, trainable=False)\n    \n    # Build the embedding layer, it is required before setting the weights of the embedding layer\n    embedding_layer.build((None,))\n    \n    # Set the weights of the embedding layer to the embedding matrix\n    embedding_layer.set_weights([embedding_matrix])\n    \n    return embedding_layer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d383e78ba2d9ef269a9102298f556596ee3f0c89"},"cell_type":"code","source":"\ndef quora_model_v1(input_shape):\n    \n    sentence_indices = Input(input_shape, dtype='int32')\n    embedding_layer = pretrained_embedding_layer(embedding_matrix)\n    \n    # Propagate sentence_indices through your embedding layer, you get back the embeddings\n    embeddings = embedding_layer(sentence_indices)   \n\n    X = Bidirectional(CuDNNLSTM(128, return_sequences=True))(embeddings)\n    X = BatchNormalization()(X)\n    X = Dropout(0.5)(X)\n    \n    X = Conv1D(32, kernel_size=(7), padding='valid', kernel_initializer='he_uniform')(X)\n    X = Activation('relu')(X)\n    X = MaxPooling1D(2)(X)\n    \n    X = Conv1D(32, kernel_size=(7), padding='valid', kernel_initializer='he_uniform')(X)\n    X = Activation('relu')(X)\n    X = MaxPooling1D(2)(X)\n    \n    X = Bidirectional(CuDNNLSTM(64, return_sequences=False))(X)\n    X = BatchNormalization()(X)\n    X = Dropout(0.4)(X)\n    \n    X = Dense(128,activation = \"relu\")(X)\n    X = Dropout(0.5)(X)\n#     X = Bidirectional(CuDNNLSTM(64, return_sequences=False))(X)\n#     X = BatchNormalization()(X)\n#     X = Dropout(0.3)(X)\n    \n    \n    X = Dense(1)(X)\n    \n    out = Activation('sigmoid', name = \"final_layer\")(X)\n    \n    # Create Model instance which converts sentence_indices into X.\n    model = Model(inputs=sentence_indices, outputs=out)\n    \n    ### END CODE HERE ###\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a32a05be75aa7db899448054e845a6ec755f148"},"cell_type":"code","source":"model = quora_model_v1((maxlen,))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f0350cc7f0fb78b26066d84d9de89edc754a5ea"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b877abf475d488419beda839303043da5a639a3a"},"cell_type":"code","source":"model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6793a9f65e177a4380dbccc69f47f5d8df76bc0a"},"cell_type":"code","source":"model.fit(train_X, train_y, epochs = 12, batch_size = 512, shuffle=True,validation_split = 0.1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f73947aee4324ac88f5a9917f4902c6be6f4725"},"cell_type":"code","source":"pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9cf3be196b522b9eed3b0a9ba14103b17e321556"},"cell_type":"code","source":"pred_glove_train_y = model.predict([train_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dcefa6c3c7a1c7d2e834648eb28387c84f09307"},"cell_type":"code","source":"train_f_score = metrics.f1_score(train_y, (pred_glove_train_y>0.4).astype(int))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dba9b2c00e354ab549522d09acce86243206eb5c"},"cell_type":"code","source":"pred_test_y = (pred_glove_test_y>0.4).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5319cc41660255ab7c4c2048d3671a5eb9176e2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}