{"cells":[{"metadata":{"_uuid":"eb721e8c304d6beeaecf948903154e25bddbb53a"},"cell_type":"markdown","source":"# LSTM + Google News Vectors\nFirst, thanks to [Dieter](https://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings) for his preprocessing with pre-trained embeddings tutorial.\n\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport re\n\ntqdm.pandas()\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train.shape)\nprint(\"Test shape : \",test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f34e4eb1e45b785c04c5d719f10fb92f6b23e4a1"},"cell_type":"code","source":"# taking a small sample (with downsampling of majority class) of the training data to speed up processing\nfrom sklearn.utils import resample\n\nsincere = train[train.target == 0]\ninsincere = train[train.target == 1]\n\ntrain = pd.concat([resample(sincere,\n                     replace = False,\n                     n_samples = len(insincere)), insincere])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f95754c2763fcb6e7b125776469ec5de4cf3f329"},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f81231d640d4ec4916570a19a1f1752aa9732de"},"cell_type":"code","source":"# text preprocessing\ncontractions = {\n\"ain't\": \"is not\",\n\"aren't\": \"are not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he will\",\n\"he'll've\": \"he he will have\",\n\"he's\": \"he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how is\",\n\"I'd\": \"I would\",\n\"I'd've\": \"I would have\",\n\"I'll\": \"I will\",\n\"I'll've\": \"I will have\",\n\"I'm\": \"I am\",\n\"I've\": \"I have\",\n\"i'd\": \"i would\",\n\"i'd've\": \"i would have\",\n\"i'll\": \"i will\",\n\"i'll've\": \"i will have\",\n\"i'm\": \"i am\",\n\"i've\": \"i have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it will\",\n\"it'll've\": \"it will have\",\n\"it's\": \"it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she will\",\n\"she'll've\": \"she will have\",\n\"she's\": \"she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so as\",\n\"that'd\": \"that would\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that is\",\n\"there'd\": \"there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there is\",\n\"they'd\": \"they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they will\",\n\"they'll've\": \"they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\"we'd\": \"we would\",\n\"we'd've\": \"we would have\",\n\"we'll\": \"we will\",\n\"we'll've\": \"we will have\",\n\"we're\": \"we are\",\n\"we've\": \"we have\",\n\"weren't\": \"were not\",\n\"what'll\": \"what will\",\n\"what'll've\": \"what will have\",\n\"what're\": \"what are\",\n\"what's\": \"what is\",\n\"what've\": \"what have\",\n\"when's\": \"when is\",\n\"when've\": \"when have\",\n\"where'd\": \"where did\",\n\"where's\": \"where is\",\n\"where've\": \"where have\",\n\"who'll\": \"who will\",\n\"who'll've\": \"who will have\",\n\"who's\": \"who is\",\n\"who've\": \"who have\",\n\"why's\": \"why is\",\n\"why've\": \"why have\",\n\"will've\": \"will have\",\n\"won't\": \"will not\",\n\"won't've\": \"will not have\",\n\"would've\": \"would have\",\n\"wouldn't\": \"would not\",\n\"wouldn't've\": \"would not have\",\n\"y'all\": \"you all\",\n\"y'all'd\": \"you all would\",\n\"y'all'd've\": \"you all would have\",\n\"y'all're\": \"you all are\",\n\"y'all've\": \"you all have\",\n\"you'd\": \"you would\",\n\"you'd've\": \"you would have\",\n\"you'll\": \"you will\",\n\"you'll've\": \"you will have\",\n\"you're\": \"you are\",\n\"you've\": \"you have\"\n}\n\nc_re = re.compile('(%s)' % '|'.join(contractions.keys()))\n\ndef expandContractions(text, c_re=c_re):\n    def replace(match):\n        return contractions[match.group(0)]\n    return c_re.sub(replace, text)\n\nfrom gensim.parsing.preprocessing import preprocess_string\nfrom gensim.parsing.preprocessing import strip_tags, strip_punctuation, strip_numeric\nfrom gensim.parsing.preprocessing import strip_multiple_whitespaces, strip_non_alphanum, remove_stopwords, strip_short\n\nCUSTOM_FILTERS = [#lambda x: x.lower(), #lowercase\n                  strip_tags, # remove html tags\n                  #strip_punctuation, # replace punctuation with space\n                  strip_multiple_whitespaces,# remove repeating whitespaces\n                  strip_non_alphanum, # remove non-alphanumeric characters\n                  #strip_numeric, # remove numbers\n                  #remove_stopwords,# remove stopwords\n                  strip_short # remove words less than minsize=3 characters long\n                 ]\ndef gensim_preprocess(docs):\n    docs = [expandContractions(doc) for doc in docs]\n    docs = [preprocess_string(text, CUSTOM_FILTERS) for text in docs]\n    docs = [' '.join(text) for text in docs]\n    return pd.Series(docs)\n\ntrain_clean = gensim_preprocess(train.question_text)\n\ngensim_preprocess(train.question_text.iloc[10:15])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80515b1714cd847c2f5a90f501d5f10290dd4af8"},"cell_type":"code","source":"# creating vocab from train dataframe\nfrom collections import Counter\nvocab = Counter()\n\ntexts = ' '.join(train_clean).split()\nvocab.update(texts)\n\nprint(len(vocab))\nprint(vocab.most_common(50))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e01724db7dc07927f0c7246edc08bfdea1c54c0e"},"cell_type":"code","source":"# load google news vectors\nfrom gensim.models import KeyedVectors\nnews_path = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(news_path, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5a8d54d8670ea0e54726ad09d28e6090a0cd5f6"},"cell_type":"code","source":"# function to check coverage of embedding vs train vocabulary\nimport operator \n\ndef check_coverage(vocab,embeddings_index):\n    a = {}\n    oov = {}\n    k = 0\n    i = 0\n    for word in tqdm(vocab):\n        try:\n            a[word] = embeddings_index[word]\n            k += vocab[word]\n        except:\n\n            oov[word] = vocab[word]\n            i += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c050f63f248cef758cd2328e044c8ee9160343a8"},"cell_type":"code","source":"# function to correct misspellings and out of vocab words\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium',\n                'Snapchat': 'social medium',\n                'quora': 'social medium',\n                'Quora': 'social medium',\n                'mediumns': 'mediums',\n                'bitcoin': 'currency',\n                'cryptocurrency': 'currency',\n                'upsc': 'union public service commission',\n                'mbbs': 'bachelor medicine',\n                'ece': 'educational credential evaluators',\n                'aiims': 'all india institute medical science',\n                'iim': 'india institute management',\n                'sbi': 'state bank india',\n                'blockchain': 'crytography',\n                'and': '',\n                'reducational':'educational',\n                'neducational':'educational',\n                'greeducational': 'greed educational',\n                'pieducational': 'educational',\n                'deducational': 'educational',\n                'Quorans': 'Quoran'   \n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)\n\n# replace numbers > 9 with #### to match embedding\ndef clean_numbers(x):\n\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ntrain_clean = train_clean.apply(lambda x: replace_typical_misspell(x))\ntrain_clean = train_clean.apply(lambda x: clean_numbers(x))\n\nvocab = Counter()\ntexts = ' '.join(train_clean).split()\nvocab.update(texts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8ff208ed3b95f97cdb89c45b7071f95d1e32a5a"},"cell_type":"code","source":"# check out of vocab words again\noov = check_coverage(vocab,embeddings_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4800e22d86ffd281b0fcfdc64ed165fac34f222"},"cell_type":"code","source":"# view top 20 oov words\noov[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5db29514870b62a7649595b54f44f30ed9009d2e"},"cell_type":"code","source":"# clean up our vocab\n# keep tokens with a min occurrence\nmin_occurrence = 5\ntokens = [k for k,c in vocab.items() if c >= min_occurrence]\nprint(len(tokens))\n\nvocab = set((' '.join(tokens)).split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87eab7bdd710519890330e59ac1a84cb56f8fe07"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils.vis_utils import plot_model\nfrom keras.models import Model\nfrom keras.layers import Dense, Flatten, Embedding, CuDNNLSTM, Bidirectional\nfrom keras.layers import Input\nfrom keras.layers.convolutional import Conv1D\nfrom keras.layers.convolutional import MaxPooling1D\n\n\n# fit a tokenizer using keras\ndef create_tokenizer(text):\n    tokenizer = Tokenizer()\n    tokenizer.fit_on_texts(text)\n    return tokenizer\ntokenizer = create_tokenizer(train_clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb7e48a6327403ae338bb97baed81ec9bd2f8f4d"},"cell_type":"code","source":"def encode_docs(tokenizer, max_length, docs):\n    # integer encode\n    encoded = tokenizer.texts_to_sequences(docs)\n    # pad sequences\n    padded = pad_sequences(encoded, maxlen = max_length, padding='post')\n    return padded","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e618afdaa9d4a72227c05341b0cc099de96d40ef"},"cell_type":"code","source":"vocab_size = len(tokenizer.word_index) + 1\nprint('Vocab Size: ', vocab_size)\nmax_length = max([len(s.split()) for s in train_clean])\nprint('Max Length: ', max_length)\nX_train = encode_docs(tokenizer, max_length, train_clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3596acf4964083f50c3323b1b3113ef861a5deda"},"cell_type":"code","source":"from keras.initializers import Constant\ntype(embeddings_index.vocab)\nEMBEDDING_DIM = 300\nMAX_NUM_WORDS = 10000\nword_index = tokenizer.word_index\nnum_words = vocab_size\nembedding_matrix = np.zeros((vocab_size, EMBEDDING_DIM))\nfor word, i in word_index.items():\n    if i > MAX_NUM_WORDS:\n        continue\n    try:\n        embedding_vector = embeddings_index.get_vector(word)\n    \n        # words not found in embedding index will be all-zeros.\n        embedding_matrix[i] = embedding_vector\n    except (KeyError):\n        continue\n        \n# load pre-trained word embeddings into an Embedding layer\nembedding_layer_google = Embedding(num_words,\n                            EMBEDDING_DIM,\n                            embeddings_initializer=Constant(embedding_matrix),\n                            input_length=max_length,\n                            trainable=True) # set trainable to true to update embeddings during training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b206fc5e99949edb01c7e3404f3a8e66cc6919fe"},"cell_type":"code","source":"# preprocess again for self trained embeddings\nfrom gensim.parsing.preprocessing import preprocess_string\nfrom gensim.parsing.preprocessing import strip_tags, strip_punctuation, strip_numeric, stem_text\nfrom gensim.parsing.preprocessing import strip_multiple_whitespaces, strip_non_alphanum, remove_stopwords, strip_short\nfrom gensim import corpora, models, similarities\n\nCUSTOM_FILTERS = [lambda x: x.lower(), #lowercase\n                  strip_tags, # remove html tags\n                  strip_punctuation, # replace punctuation with space\n                  strip_multiple_whitespaces,# remove repeating whitespaces\n                  strip_non_alphanum, # remove non-alphanumeric characters\n                  strip_numeric, # remove numbers\n                  remove_stopwords,# remove stopwords\n                  strip_short, # remove words less than minsize=3 characters long\n                  stem_text,\n                 ]\ndef ngram_preprocess(docs):\n    # clean text\n    docs = [expandContractions(doc) for doc in docs]\n    docs = [preprocess_string(text, CUSTOM_FILTERS) for text in docs]\n    # create the bigram and trigram models\n    bigram = models.Phrases(docs, min_count=1, threshold=1)\n    trigram = models.Phrases(bigram[docs], min_count=1, threshold=1)  \n    # phraser is faster\n    bigram_mod = models.phrases.Phraser(bigram)\n    trigram_mod = models.phrases.Phraser(trigram)\n    # apply to docs\n    docs = trigram_mod[bigram_mod[docs]]\n    docs = [' '.join(text) for text in docs]\n    return pd.Series(docs)\n\ntrain_ngram = ngram_preprocess(train.question_text)\ntrain_ngram[43]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b07ce48359e46be707d2ab11714d72918d3d5e9b"},"cell_type":"code","source":"# define ngram vocab\nngram_vocab = Counter()\nngram_texts = ' '.join(train_ngram).split()\nngram_vocab.update(ngram_texts)\nprint(len(ngram_vocab))\nprint(ngram_vocab.most_common(50))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ef8f818a2bba429d18d5c2ce526eee5e6ebb449"},"cell_type":"code","source":"# keep tokens with min occurrence\nmin_ngram = 3\nngram_tokens = [k for k, c in ngram_vocab.items() if c >= min_ngram]\nprint(len(ngram_tokens))\n\n# get rid of duplicate words\nvocab = set((' '.join(tokens)).split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65a90b73819ec16c1a25d3314fd61b2aac6403f9"},"cell_type":"code","source":"# fit tokenizer\nngram_tokenizer = create_tokenizer(train_ngram)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76a1cfe49ceb9485e45fccf405b76234be615121"},"cell_type":"code","source":"ngram_vocab_size = len(ngram_tokenizer.word_index) + 1\nprint('Vocab Size: ', ngram_vocab_size)\nngram_max_length = max([len(s.split()) for s in train_ngram])\nprint('Max Length: ', ngram_max_length)\nX_ngram_train = encode_docs(ngram_tokenizer, ngram_max_length, train_ngram)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43bcbf48e6e61a2ab5a923537d31012cd8743b3e"},"cell_type":"code","source":"from keras.layers import Input, Dense, Flatten, Embedding, CuDNNLSTM, Bidirectional, concatenate\nfrom keras.layers import SpatialDropout1D, GlobalMaxPool1D, Dropout, GlobalAveragePooling1D, GlobalMaxPooling1D\nfrom keras.layers.convolutional import Conv1D\nfrom keras.layers.convolutional import MaxPooling1D\n\n# define and fit model on dataset\ny_train = train.target\n\ndef define_model(max_length, ngram_max_length):\n    inp1 = Input(shape=(max_length, ))\n    x = (embedding_layer_google)(inp1)\n    x = SpatialDropout1D(0.3)(x)\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n    x = Flatten()(x)\n    \n    inp2 = Input(shape=(ngram_max_length, ))\n    y = Embedding(ngram_vocab_size, 100, input_length=ngram_max_length)(inp2)\n    y = Conv1D(filters=32, kernel_size=8, activation='relu')(y)\n    y = MaxPooling1D(pool_size=2)(y)\n    y = Flatten()(y)\n    \n    z = concatenate([x, y])\n    z = Dense(64, activation='relu')(z)\n    z = Dense(1, activation='sigmoid')(z)\n    model = Model(inputs=[inp1, inp2], outputs=z)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    #summarize defined model\n    model.summary()\n    plot_model(model, to_file='model.png', show_shapes=True)\n    return model\n\nmodel = define_model(max_length, ngram_max_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bce5e22ffdd1cc4d420f908e29d20668207c2531"},"cell_type":"code","source":"def fit_model(X_train, X_ngram_train, y_train):\n    model.fit([X_train, X_ngram_train],\n              y_train,\n              epochs=5,\n              verbose=1,\n              shuffle=True,\n              validation_split=0.1,\n              class_weight={1:0.6, 0:0.4})\n    return model\n\nmodel = fit_model(X_train, X_ngram_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1f3ae0de3517a92d23de3d67bfc4b676f835560"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c97ebfdd9210d7a8e6bdeafe66de40d0c53e100"},"cell_type":"code","source":"# prepare test data inp1\ntest_clean = gensim_preprocess(test.question_text)\ntest_clean = test_clean.apply(lambda x: replace_typical_misspell(x))\ntest_clean = test_clean.apply(lambda x: clean_numbers(x))\npred = encode_docs(tokenizer, max_length, test_clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf4b8fa6f860d2c54bb30c7f394e88a36441c870"},"cell_type":"code","source":"# prepare test data inp2\ntest_ngram = ngram_preprocess(test.question_text)\nngram_pred = encode_docs(ngram_tokenizer, ngram_max_length, test_ngram)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7220a661a5d08d38eefed111d040cd71f74035e2"},"cell_type":"code","source":"# predict on test data\nprediction = model.predict([pred, ngram_pred], verbose=1)\nprediction = [1 if proba >=0.5 else 0 for proba in prediction]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78f83df345d3896addf57dedc4b4a7d8b1422657"},"cell_type":"code","source":"submission = pd.DataFrame({'qid':test.qid, 'prediction':prediction})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"476e7c51bb1cae2398cb437da48a09dab1b4f35f"},"cell_type":"code","source":"submission.prediction.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78da8c0927f2cf87f43b4bdbda822926410a80cd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}