{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from __future__ import division\nimport os\nimport random\nimport numpy as np\nimport tensorflow as tf\nfrom keras import backend as K\n\nimport pandas as pd\n\nfrom keras.layers import Input, Dropout, Dense, concatenate,  Embedding, Flatten, Activation, CuDNNLSTM,  Lambda\nfrom keras.layers import Conv1D, Bidirectional, SpatialDropout1D, BatchNormalization, multiply\nfrom keras.layers import GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras import optimizers, callbacks, regularizers\nfrom keras.models import Model\n\n\nfrom collections import defaultdict\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nfrom sklearn.metrics import log_loss\n\nimport re\n\nimport gc\nimport time\nimport nltk\n\nfrom nltk.stem import WordNetLemmatizer\nwordnet_lemmatizer = WordNetLemmatizer()\nfrom nltk.corpus import stopwords\nstop_words = set(stopwords.words('english'))\n\nfrom keras.preprocessing.text import Tokenizer\nfrom sklearn.model_selection import train_test_split\nimport operator \n\nimport matplotlib.pyplot as plt\n\nPATH = '../input/'\nEMBEDDINGS_PATH = '../input/embeddings/'\nWEIGHTS_PATH = './w0.h5'\nMAX_TEXT_LENGTH = 40\nEMBEDDING_SIZE  = 300\n\n\n\n\ndef check_coverage(vocab,embeddings_index):\n    a = {}\n    oov = {}\n    k = 0\n    i = 0\n    for word in vocab:\n        try:\n            a[word] = embeddings_index[word]\n            k += vocab[word]\n        except:\n\n            oov[word] = vocab[word]\n            i += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n\n\n\ncontraction_mapping = {u\"ain't\": u\"is not\", u\"aren't\": u\"are not\",u\"can't\": u\"cannot\", u\"'cause\": u\"because\",\n                       u\"could've\": u\"could have\", u\"couldn't\": u\"could not\", u\"didn't\": u\"did not\",\n                       u\"doesn't\": u\"does not\", u\"don't\": u\"do not\", u\"hadn't\": u\"had not\",\n                       u\"hasn't\": u\"has not\", u\"haven't\": u\"have not\", u\"he'd\": u\"he would\",\n                       u\"he'll\": u\"he will\", u\"he's\": u\"he is\", u\"how'd\": u\"how did\", u\"how'd'y\": u\"how do you\",\n                       u\"how'll\": u\"how will\", u\"how's\": u\"how is\",  u\"I'd\": u\"I would\",\n                       u\"I'd've\": u\"I would have\", u\"I'll\": u\"I will\", u\"I'll've\": u\"I will have\",\n                       u\"I'm\": u\"I am\", u\"I've\": u\"I have\", u\"i'd\": u\"i would\", u\"i'd've\": u\"i would have\",\n                       u\"i'll\": u\"i will\",  u\"i'll've\": u\"i will have\",u\"i'm\": u\"i am\", u\"i've\": u\"i have\",\n                       u\"isn't\": u\"is not\", u\"it'd\": u\"it would\", u\"it'd've\": u\"it would have\",\n                       u\"it'll\": u\"it will\", u\"it'll've\": u\"it will have\",u\"it's\": u\"it is\",\n                       u\"let's\": u\"let us\", u\"ma'am\": u\"madam\", u\"mayn't\": u\"may not\",\n                       u\"might've\": u\"might have\",u\"mightn't\": u\"might not\",u\"mightn't've\": u\"might not have\",\n                       u\"must've\": u\"must have\", u\"mustn't\": u\"must not\", u\"mustn't've\": u\"must not have\",\n                       u\"needn't\": u\"need not\", u\"needn't've\": u\"need not have\",u\"o'clock\": u\"of the clock\",\n                       u\"oughtn't\": u\"ought not\", u\"oughtn't've\": u\"ought not have\", u\"shan't\": u\"shall not\", \n                       u\"sha'n't\": u\"shall not\", u\"shan't've\": u\"shall not have\", u\"she'd\": u\"she would\",\n                       u\"she'd've\": u\"she would have\", u\"she'll\": u\"she will\", u\"she'll've\": u\"she will have\",\n                       u\"she's\": u\"she is\", u\"should've\": u\"should have\", u\"shouldn't\": u\"should not\",\n                       u\"shouldn't've\": u\"should not have\", u\"so've\": u\"so have\",u\"so's\": u\"so as\",\n                       u\"this's\": u\"this is\",u\"that'd\": u\"that would\", u\"that'd've\": u\"that would have\",\n                       u\"that's\": u\"that is\", u\"there'd\": u\"there would\", u\"there'd've\": u\"there would have\",\n                       u\"there's\": u\"there is\", u\"here's\": u\"here is\",u\"they'd\": u\"they would\", \n                       u\"they'd've\": u\"they would have\", u\"they'll\": u\"they will\", \n                       u\"they'll've\": u\"they will have\", u\"they're\": u\"they are\", u\"they've\": u\"they have\", \n                       u\"to've\": u\"to have\", u\"wasn't\": u\"was not\", u\"we'd\": u\"we would\",\n                       u\"we'd've\": u\"we would have\", u\"we'll\": u\"we will\", u\"we'll've\": u\"we will have\", \n                       u\"we're\": u\"we are\", u\"we've\": u\"we have\", u\"weren't\": u\"were not\",\n                       u\"what'll\": u\"what will\", u\"what'll've\": u\"what will have\", u\"what're\": u\"what are\",\n                       u\"what's\": u\"what is\", u\"what've\": u\"what have\", u\"when's\": u\"when is\",\n                       u\"when've\": u\"when have\", u\"where'd\": u\"where did\", u\"where's\": u\"where is\",\n                       u\"where've\": u\"where have\", u\"who'll\": u\"who will\", u\"who'll've\": u\"who will have\",\n                       u\"who's\": u\"who is\", u\"who've\": u\"who have\", u\"why's\": u\"why is\", u\"why've\": u\"why have\",\n                       u\"will've\": u\"will have\", u\"won't\": u\"will not\", u\"won't've\": u\"will not have\",\n                       u\"would've\": u\"would have\", u\"wouldn't\": u\"would not\", u\"wouldn't've\": u\"would not have\",\n                       u\"y'all\": u\"you all\", u\"y'all'd\": u\"you all would\",u\"y'all'd've\": u\"you all would have\",\n                       u\"y'all're\": u\"you all are\",u\"y'all've\": u\"you all have\",u\"you'd\": u\"you would\",\n                       u\"you'd've\": u\"you would have\", u\"you'll\": u\"you will\", u\"you'll've\": u\"you will have\",\n                       u\"you're\": u\"you are\", u\"you've\": u\"you have\", u\"didnt\": u\"did not\" }\n\ndef remove_special_chars(w):\n    for i, j in [ (u\"é\", u\"e\"), (u\"ē\", u\"e\"), (u\"è\", u\"e\"), (u\"ê\", u\"e\"), (u\"à\", u\"a\"),\n                 (u\"â\", u\"a\"), (u\"ô\", u\"o\"), (u\"ō\", u\"o\"), (u\"ü\", u\"u\"), (u\"ï\", u\"i\"),\n                 (u\"ç\", u\"c\"), (u\"\\xed\", u\"i\")]:\n        x = re.sub(i, j, w)\n        if x in embeddings_index:\n            return x\n        \n    return w\n\ndef lower(w):\n    x = w.lower()\n    if x in embeddings_index:\n        return x\n    else:\n        return w\n    \ndef keep_alpha_num(w):\n    x = re.sub(u\"[^a-z\\s0-9]\", u\" \", w)\n    x = re.sub( u\"\\s+\", u\" \", x ).strip()\n    return x\n\n\ndef keep_only_alpha(w):\n    x = re.sub(u\"[^a-z]\", u\" \", w)\n    x = re.sub( u\"\\s+\", u\" \", x ).strip()\n    return x\n\ndef preprocess( text ):\n    text = re.sub( u\"\\s+\", u\" \", text ).strip()\n    \n    text = re.sub( u\"\\[math\\].*\\[\\/math\\]\", u\" math \", text) \n    text = re.sub( u\"\\S*@\\S*\\.\\S*\", u\" email \", text) \n    \n    #replace any integer or real number by the word \"number\"\n    text = u\" \".join( re.sub(u\"^\\d+(?:[.,]\\d*)?$\", u\"number\", w)  for w in text.split(\" \"))\n    \n    \n    specials = [u\"’\", u\"‘\", u\"´\", u\"`\", u\"\\u2019\"]\n    for s in specials:\n        text = text.replace(s, u\"'\")# normalize \" ' \", also will be helpful for contractions\n        \n    text = u\" \".join( [contraction_mapping[w] if w in contraction_mapping else w for w in text.split(\" \") ] ) \n    \n    text = u\" \".join( [w if w in embeddings_index else remove_special_chars(w).strip() for w in text.split(\" \")] ) \n    \n    text = u\" \".join( [w if w in embeddings_index else lower(w).strip() for w in text.split(\" \")] )\n    \n    text = u\" \".join( [w if w in embeddings_index else keep_alpha_num(w).strip() for w in text.split(\" \")] )\n    \n    text = u\" \".join( [w if w in embeddings_index else keep_only_alpha(w).strip() for w in text.split(\" \")] )\n    \n    text = text.split(' ')[:MAX_TEXT_LENGTH]\n \n    return ' '.join(text)\n\ndef embeddingNN(data,trainable=True, seed=42):                                             \n    np.random.seed(seed)\n\n    emb_inpt  = Input( shape=[data.shape[1]], name='emb_inpt')   \n    \n    #dme\n    if len(embedding_weights.shape)==3:\n        x1 = Embedding(len( encoding_dc )+1, embedding_weights.shape[1], weights=[embedding_weights[:,:,0]], trainable=trainable) (emb_inpt)\n        x2 = Embedding(len( encoding_dc )+1, embedding_weights.shape[1], weights=[embedding_weights[:,:,1]], trainable=trainable) (emb_inpt)\n        print (x1.shape, x2.shape)\n        x = Lambda( lambda x: K.stack( [x[0],x[1]], axis=-1 ) )([x1,x2]) \n        print (x.shape)\n        x = CDME_Block(x, MAX_TEXT_LENGTH, n_emb=embedding_weights.shape[-1])\n    else:\n        x = Embedding(len( encoding_dc )+1, embedding_weights.shape[1], weights=[embedding_weights], trainable=trainable) (emb_inpt)\n        \n    x = CuDNNLSTM(64, return_sequences=True) (x)   \n    x = GlobalMaxPooling1D()(x)  \n\n\n    x= Dense(128, trainable=not trainable)(x)\n    x = Activation('relu')(x)\n    \n    x= Dense(1, trainable=not trainable)(x)\n    x = Activation('sigmoid')(x)\n    \n    model = Model([emb_inpt],x)\n\n    return model\n\n\ndef run_model(lr=1e-3, bs=2048):    \n    predictions_test   = pd.DataFrame()\n    predictions_train  = pd.DataFrame()\n    for seed in range(3):\n        es = callbacks.EarlyStopping( patience=2 )\n        mc = callbacks.ModelCheckpoint( filepath=WEIGHTS_PATH, monitor='val_loss', mode='min', save_best_only=True )\n\n        model = embeddingNN(X_test_emb, trainable=False, seed=seed)\n        \n        optimizer = optimizers.Adam(lr=lr)\n        model.compile(loss=\"binary_crossentropy\", optimizer=optimizer)\n\n        model.fit(  X_train_emb, y_train, validation_data=(X_test_emb, y_test), callbacks=[es, mc],\n                     batch_size=bs, epochs=1000, verbose=2 )\n        ###############################\n        es = callbacks.EarlyStopping( patience=2 )\n        model = embeddingNN(X_test_emb, trainable=True, seed=seed)\n        model.load_weights(WEIGHTS_PATH)\n        optimizer = optimizers.Adam(lr=lr/10.)\n        model.compile(loss=\"binary_crossentropy\", optimizer=optimizer)\n\n        model.fit(  X_train_emb, y_train, validation_data=(X_test_emb, y_test), callbacks=[es, mc],\n                     batch_size=2048, epochs=1000, verbose=2 )\n        #######################################\n        model.load_weights(WEIGHTS_PATH)\n\n        p = model.predict(X_test_emb, batch_size=4096)\n        predictions_test[str(seed)] = p.flatten()\n\n        p = model.predict(X_train_emb, batch_size=4096)\n        predictions_train[str(seed)] = p.flatten()\n\n        print ( 'BAGGING SCORE Test: ' , log_loss(y_test,  predictions_test.mean(axis=1), eps = 1e-7) )\n        print ( 'BAGGING SCORE Train: ', log_loss(y_train, predictions_train.mean(axis=1), eps = 1e-7) )\n        \n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"full_data = pd.read_csv(PATH+'train.csv',  encoding='utf-8', engine='python')\nfull_data['question_text'].fillna(u'unknownstring', inplace=True)\n\nprint (full_data.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Meta-Embedding\nDepending on the given task, some pretrained embeddings may be better than others. Combining them means more diversity, complementarity and an increase in the coverage. Meta-Embedding can be seen as en ensemble method.\n\nWe are provided with four different pretrained embeddings.\n\nGoogleNews-vectors-negative300 - https://code.google.com/archive/p/word2vec/\n\nglove.840B.300d - https://nlp.stanford.edu/projects/glove/\n\nparagram_300_sl999 - https://cogcomp.org/page/resource_view/106\n\nwiki-news-300d-1M - https://fasttext.cc/docs/en/english-vectors.html\n\n\nIn this kernel, we will see how to combine different embeddings using different approaches. To keep code easy to read, I will use only 2 embeddings (glove and paragram)\n\nFirst we will check the performance of glove and paragram individualy, then we will try the concatenation, average and dynamic meta embedding approaches  \n\nOur dataset has a shape of (n, embedding size)\n\nConcatenation : for each word, concatenate the embeddings, the input shape for NN is (n, 2 * embedding size)\n\nAverage       : for each word, average the embeddings, the input shape for NN is (n, 1 * embedding size)\n\nDme           : for each word, let the NN decide how to combine the embeddings, the input shape for NN is (n, embedding size, 2)\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"#our word index will contain now the union of the vocabulary from glove and paragram\n\ndef load_glove_words():\n    \n    def get_coefs(word,*arr): return word, 1\n    \n    EMBEDDING_FILE = EMBEDDINGS_PATH+'glove.840B.300d/glove.840B.300d.txt'        \n    embeddings_dict = dict()        \n    embeddings_dict.update( dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE)) )\n    \n    EMBEDDING_FILE = EMBEDDINGS_PATH+'paragram_300_sl999/paragram_300_sl999.txt'\n    embeddings_dict.update(  dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100) )\n    \n\n    return embeddings_dict\n\nembeddings_index = load_glove_words()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_test, y_train, y_test = train_test_split(  full_data.question_text.values, full_data.target.values, \n                                                    shuffle =True, test_size=0.5, random_state=42)\n\nX_train = np.array( [preprocess(x) for x in X_train] )\nX_test  = np.array( [preprocess(x) for x in X_test] )\n\nword_frequency_dc=defaultdict(np.uint32)\ndef word_count(text):\n    text = text.split(' ')\n    for w in text:\n        word_frequency_dc[w]+=1\n\nfor x in X_train:\n    word_count(x) \n\nencoding_dc = dict()\nlabelencoder=1\nfor key in word_frequency_dc:\n    if word_frequency_dc[key]>1:\n        encoding_dc[key]=labelencoder\n        labelencoder+=1\n    \n\ncheck_coverage(word_frequency_dc,embeddings_index)\nprint ('number of unique words in the dataset after preprocessing : ', len(word_frequency_dc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef preprocess_keras(text):\n    \n    def get_encoding(w):\n        if w in encoding_dc:\n            return encoding_dc[w]\n        return 0\n    \n    x = [ get_encoding(w) for w in text.split(' ') ]\n    x = x + (MAX_TEXT_LENGTH-len(x))*[0]\n    return x\nX_train_emb = np.array( [ preprocess_keras(x) for x in X_train ] )\nX_test_emb  = np.array( [ preprocess_keras(x) for x in X_test ]  )\nprint ( X_train_emb.shape, X_test_emb.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EMBEDDING_SIZE = 300\n\ndef get_embeddings( word_index , method):\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    \n    if method == 'glove':\n        EMBEDDING_FILE = EMBEDDINGS_PATH+'glove.840B.300d/glove.840B.300d.txt'\n        embeddings = { o.split(\" \")[0]:np.asarray(o.split(\" \")[1:], dtype='float32') for o in open(EMBEDDING_FILE) if o.split(\" \")[0] in word_index }\n    \n    if method == 'paragram':\n        EMBEDDING_FILE = EMBEDDINGS_PATH+'paragram_300_sl999/paragram_300_sl999.txt'  \n        embeddings = { o.split(\" \")[0]:np.asarray(o.split(\" \")[1:], dtype='float32')\\\n                      for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore')\\\n                      if ( (len(o)>100) & (o.split(\" \")[0] in word_index) ) }\n    \n        \n    temp = np.stack(embeddings.values())\n    mean, std = temp.mean(), temp.std()\n  \n    embedding_weights    = np.random.normal(mean, std, (len(word_index)+1,  EMBEDDING_SIZE ) ).astype(np.float32)\n\n    for word, i in word_index.items():\n        if (word in embeddings):\n            embedding_weights[i] = embeddings.get(word)\n\n    return embedding_weights, embeddings\n            \n            \ndef load_embeddings(word_index, method='glove'):\n    # method is either : 'glove', 'paragram', 'concat', 'avg', 'dme'\n    \n    if method in [ 'glove' , 'paragram']:\n        return get_embeddings( word_index, method )[0]\n    else:\n        embedding_glove, glove_index       = get_embeddings( word_index, method='glove' )\n        embedding_paragram , paragram_index   = get_embeddings( word_index, method='paragram' ) \n        \n        if method == 'concat':\n            return np.hstack( [embedding_glove, embedding_paragram] )\n        if method == 'avg':\n            return (embedding_glove + embedding_paragram) / 2.0           \n        if method == 'dme':\n            return np.stack( [embedding_glove, embedding_paragram], axis=-1 )\n        \n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## DME implementation\n\nn_emb=2 in our case because we are using two embedding\n\n\nthis keras layer is doing dme, the idea is to let the network compute the optimal weight to assign to each embedding for each word :\n\n### step 1 : get two weights vectors using an lstm and make sure values are between 0 and 1\n\nx = CuDNNLSTM(n_emb,return_sequences = True)(x)\n\nx = Activation('sigmoid')(x), so that weights are between 0 and 1\n\n### step 2 : multiply each embedding by its weight vector\n\nx = multiply([init, x])\n\n### step 3 : sum the weighted embeddings, so dme is just a weighted average of the embeddings\n\nout = Lambda(lambda x: K.sum(x, axis=-1))(x)\n\n### Dme is just an attention layer, it pays more attention to what embedding is important using the weights\n\n\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.layers import Activation\nfrom keras.layers import multiply, Lambda, Reshape\nimport keras.backend as K\n#https://www.kaggle.com/c/quora-insincere-questions-classification/discussion/71778\ndef CDME_Block(inp, maxlen, n_emb):\n    \"\"\"\n    # inp = tensor of shape (?,maxlen,embedding dim,n_emb)) n_emb is number of embedding matrices\n    # out = tensor of shape (?,maxlen,embedding dim)\n    \"\"\"\n    init = inp\n    x = Reshape((maxlen,-1))(inp)\n    \n    x = CuDNNLSTM(n_emb,return_sequences = True)(x)\n    x = Activation('sigmoid')(x)\n    x = Reshape((maxlen,1,n_emb))(x)\n    x = multiply([init, x])\n    out = Lambda(lambda x: K.sum(x, axis=-1))(x)\n    return out\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_weights = load_embeddings(encoding_dc, method='glove')\nprint (embedding_weights.shape)\nrun_model(lr=5e-3, bs=2048)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_weights = load_embeddings(encoding_dc, method='paragram')\nprint (embedding_weights.shape)\nrun_model(lr=5e-3, bs=2048)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_weights = load_embeddings(encoding_dc, method='concat')\nprint (embedding_weights.shape)\nrun_model(lr=5e-3, bs=2048)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_weights = load_embeddings(encoding_dc, method='avg')\nprint (embedding_weights.shape)\nrun_model(lr=5e-3, bs=2048)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_weights = load_embeddings(encoding_dc, method='dme')\nprint (embedding_weights.shape)\nrun_model(lr=5e-3, bs=2048)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"glove : 0.09977\n\nparagram : 0.10499\n\nparagram score is really bad, we may expect that concatenation and average will not work and dme to be as good as glove\n\n\nconcat : 0.101157\n\navg : 0.100724\n\ndme : 0.100134, in worst case, it should perform as good as the best of the used embeddings"},{"metadata":{},"cell_type":"markdown","source":"## References\n\n\nhttps://www.aclweb.org/anthology/P16-1128 Introduction to Meta Embedding\n\nhttps://arxiv.org/pdf/1804.07983.pdf Dme\n\nhttps://www.aclweb.org/anthology/N18-2031 Theoretical explanation on the use of average embeddings\n\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}