{"cells":[{"metadata":{"_uuid":"2dcfb8978bec37e35b8c6d36b36f4882e5bd97ae"},"cell_type":"markdown","source":"#### This kernel is based on Google paper \"Attention is all you need\". Original paper dosen't use any LSTM and CNN, but HERE I also add one Bidirectional-LSTM layer, because I find that LSTM is great for this datasets.\nThanks to this kernel for Attention layer implement: https://www.kaggle.com/shujian/different-embeddings-with-attention-fork-fork, you should also need to check it out.\nYou can also check out original paper: https://arxiv.org/abs/1706.03762\n#### Here we go!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import some libaries\nimport numpy as np\nimport pandas as pd\nimport os\nimport time\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom matplotlib import style\nimport seaborn as sns\nimport tensorflow as tf\nfrom sklearn import metrics\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom sklearn.model_selection import train_test_split\nfrom nltk.tokenize import word_tokenize\nfrom scikitplot.metrics import plot_confusion_matrix\n\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.layers import Add, Bidirectional, CuDNNLSTM, Dense, Input, Embedding,merge, BatchNormalization, Reshape\nfrom keras.models import Model","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# load train and test datasets\ntrain = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(\"Train datasets shape:\", train.shape)\nprint(\"Test datasets shape:\", test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce1c0f7847e40e3549d1d3093d4e6abcddaddc0b"},"cell_type":"code","source":"# Here I will split the data to train and validation data\ntrain_data, validation_data = train_test_split(train, test_size=.1, random_state=1234)\n\n# Here I will use Tokenizer to extract the keyword vector as baseline\n# I will use train data to fit the Tokenizer, then use this Tokenizer to extract the validation data\nmax_length = 100\nmax_features = 50000\ntoken = Tokenizer(num_words=max_features)\ntoken.fit_on_texts(list(np.asarray(train_data.question_text)))\nxtrain = token.texts_to_sequences(np.asarray(train_data.question_text))\nxvalidate = token.texts_to_sequences(np.asarray(validation_data.question_text))\nxtest = token.texts_to_sequences(np.asarray(test.question_text))\n\n# Because Tokenizer will split the sentence, for some sentence are smaller,\n# so we have to pad the missing position\nxtrain = pad_sequences(xtrain, maxlen=max_length)\nxvalidate = pad_sequences(xvalidate, maxlen=max_length)\nxtest = pad_sequences(xtest, maxlen=max_length)\n\nytrain = train_data.target\nyvaliate = validation_data.target","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f43fae5a40b963c11eecea0f97938036a8f17417"},"cell_type":"code","source":"# Here I write a helper function to evaluate model\ndef evaluate(y, pred):\n    f1_list = list()\n    thre_list = np.arange(0.1, 0.501, 0.01)\n    for thresh in thre_list:\n        thresh = np.round(thresh, 2)\n        f1 = metrics.f1_score(y, (pred>thresh).astype(int))\n        f1_list.append(f1)\n        print(\"F1 score at threshold {0} is {1}\".format(thresh, f1))\n    #return f1_list\n    plot_confusion_matrix(y, np.array(pd.Series(pred.reshape(-1,)).map(lambda x:1 if x>thre_list[np.argmax(f1_list)] else 0)))\n    print('Best Threshold: ',thre_list[np.argmax(f1_list)])\n    return thre_list[np.argmax(f1_list)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ed20dc2eb32807452350814ef591444ab5074c1"},"cell_type":"markdown","source":"#### This is Attention layer implement."},{"metadata":{"trusted":true,"_uuid":"97b2e45d008067f37da7d76454be3e183c9a3f4f"},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb6c3622723c9b6f58f9c72cf921ee3a69dcd7dd"},"cell_type":"markdown","source":"#### Here is Glove embeddings."},{"metadata":{"trusted":true,"_uuid":"8a06c7138e78f784ee637e6c48ede86771983a7e"},"cell_type":"code","source":"em_file = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word, *arr):\n    return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*d.split(' ')) for d in open(em_file))\n\nall_embs = np.stack(embedding_index.values())\nem_mean, em_std = all_embs.mean(), all_embs.std()\nem_size = all_embs.shape[1]\n\nword_index = token.word_index\nnb_words = min(max_features, len(word_index))\nem_matrix = np.random.normal(em_mean, em_std, (nb_words, em_size))\n# loop for every word\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    em_v = embedding_index.get(word)\n    if em_v is not None:\n        em_matrix[i] = em_v\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"520633c1572e33f47b6f5ab768eefef1b8683ca7"},"cell_type":"code","source":"\"\"\"Here is attention model parameters\"\"\"\n# You can tune attention layers numbers and parallism attention\natten_layers = 1    # How many attention block to be used.\nnum_att = 10        # How many parallism attention layer to be used.\nlstm_units = 128    # How many LSTM units to be used.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cdea223eea7ebd244a742263e745862ad049c616"},"cell_type":"code","source":"### Here I will build a Attention model\ndef attention_model(em_matrix, atten_layers=atten_layers, num_att=num_att, lstm_units=lstm_units):\n    inp = Input(shape=(max_length, ))\n    x = Embedding(max_features, em_matrix.shape[1], weights=[em_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNLSTM(lstm_units, return_sequences=True))(x)\n\n    # Here I build a bisic Attention block\n    def attention_block(layer, atten_len=max_length, num_multi=num_att, dense_units=lstm_units*2):\n        atten_list = []\n        for _ in range(num_multi):\n            atten_list.append(Attention(atten_len)(layer))\n        add_layer = Add()(atten_list)\n        add_layer = BatchNormalization()(add_layer)\n        dense_layer = Dense(dense_units, activation='relu')(add_layer)\n        return Add()([add_layer, dense_layer])   # Residual add layer\n\n    for j in range(atten_layers):\n        if j ==0:\n            x = attention_block(x)\n        else:\n            x = Reshape((lstm_units*2, 1))(x)\n            x = attention_block(x, atten_len=lstm_units*2)\n    \n    x = Dense(64, activation='relu')(x)\n    out = Dense(1, activation='sigmoid')(x)\n\n    model = Model(inp, out)\n\n    model.summary()\n\n    model.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer='adam')\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a39698bc5060292e84b209a7204b0f5e76e4710"},"cell_type":"code","source":"# I build this model based on wide&deep model structure idea\nfrom keras.layers import concatenate\ndef att_new_model(em_matrix, atten_layers=atten_layers, num_att=num_att, lstm_units=lstm_units):\n    inp = Input(shape=(max_length, ))\n    x = Embedding(max_features, em_matrix.shape[1], weights=[em_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNLSTM(lstm_units, return_sequences=True))(x)\n\n    # Here I build a bisic Attention block\n    def attention_block(layer, atten_len=max_length, num_multi=num_att, dense_units=lstm_units*2):\n        add_layer_list = []\n        for j in range(num_att):\n            atten_list = list()\n            for _ in range(num_multi):\n                atten_list.append(Attention(atten_len)(layer))\n            add_layer_list.append(Add()(atten_list))\n        add_layer = concatenate(add_layer_list)\n        add_layer = BatchNormalization()(add_layer)\n        dense_layer = Dense(dense_units, activation='relu')(add_layer)\n        return dense_layer\n        # return Add()([add_layer, dense_layer])   # Residual add layer\n\n#     for j in range(atten_layers):\n#         if j ==0:\n#             x = attention_block(x)\n#         else:\n#             x = Reshape((lstm_units*2, 1))(x)\n#             x = attention_block(x, atten_len=lstm_units*2)\n    x = attention_block(x)\n    x = Dense(64, activation='relu')(x)\n    out = Dense(1, activation='sigmoid')(x)\n\n    model = Model(inp, out)\n\n    model.summary()\n\n    model.compile(loss='binary_crossentropy', metrics=['accuracy'], optimizer='adam')\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"0f119a7aa900f28ea5dcc79bfa61c2dc911b0fc9"},"cell_type":"code","source":"# model = att_new_model(em_matrix=em_matrix, atten_layers=10, lstm_units=64, num_att=10)\n# model.fit(xtrain, ytrain, epochs=2, batch_size=512, validation_data=(xvalidate, yvaliate), verbose=1)\n\n# pred_vali_glove = model.predict(xvalidate, batch_size=1024)\n# best_thre = evaluate(yvaliate, pred_vali_glove)\n\n# pred_test_glove = model.predict(xtest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d28ee03bb40eb38c97dcf624f9b651814957741","scrolled":false},"cell_type":"code","source":"model = attention_model(em_matrix=em_matrix, atten_layers=1, lstm_units=64, num_att=10)\nmodel.fit(xtrain, ytrain, epochs=2, batch_size=512, validation_data=(xvalidate, yvaliate), verbose=1)\n\npred_vali_glove = model.predict(xvalidate, batch_size=1024)\nbest_thre = evaluate(yvaliate, pred_vali_glove)\n\npred_test_glove = model.predict(xtest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1e0f8ad88e57091d96decf07060aebda7d157a4"},"cell_type":"code","source":"del embedding_index, all_embs, word_index, em_matrix\nimport gc\ngc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b335f59759ec0227b9ea8acd7c3b306f373c25fe"},"cell_type":"markdown","source":"#### Wiki-news"},{"metadata":{"_uuid":"ec254fd8bc7f74fdc51925ee958cfc3ca769a0dc"},"cell_type":"markdown","source":"#### Because I find that no matter what deep learning model used, that for this wiki embedding just is bad! Not use this embedding for ensemble model!"},{"metadata":{"trusted":true,"_uuid":"3fc3a118fbcc75da8830b395d7921f7d509f5cbc"},"cell_type":"code","source":"# em_file = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n# def get_coefs(word, *arr):\n#     return word, np.asarray(arr, dtype='float32')\n# embedding_index = dict(get_coefs(*o.split(\" \")) for o in open(em_file) if len(o)>100)\n\n# all_embs = np.stack(embedding_index.values())\n# em_mean, em_std = all_embs.mean(), all_embs.std()\n# em_size = all_embs.shape[1]\n\n# word_index = token.word_index\n# nb_words = min(max_features, len(word_index))\n# em_matrix = np.random.normal(em_mean, em_std, (nb_words, em_size))\n# # loop for every word\n# for word, i in word_index.items():\n#     if i >= max_features: continue\n#     em_v = embedding_index.get(word)\n#     if em_v is not None:\n#         em_matrix[i] = em_v","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af18ed85fc110f1560be79445b623ba9b4071460"},"cell_type":"code","source":"# model = attention_model(em_matrix=em_matrix)\n# model.fit(xtrain, ytrain, epochs=2, batch_size=512, validation_data=(xvalidate, yvaliate), verbose=1)\n\n# pred_vali_wiki = model.predict(xvalidate, batch_size=1024)\n# best_thre = evaluate(yvaliate, pred_vali_wiki)\n\n# pred_test_wiki = model.predict(xtest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d41a1c93cdef7123710899ccd812835f636e302d"},"cell_type":"code","source":"# del embedding_index, all_embs, word_index, em_matrix\n# gc.collect()\n# time.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5dd19a4cf4056b1c239caa672cfbf4699471a6c6"},"cell_type":"markdown","source":"#### Paragram"},{"metadata":{"trusted":true,"_uuid":"c4c5786468a97ef68d701e133e00c7435d8bafdc"},"cell_type":"code","source":"em_file = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word, *arr):\n    return word, np.asarray(arr, dtype='float32')\nembedding_index = dict(get_coefs(*o.split(\" \")) for o in open(em_file, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embedding_index.values())\nem_mean, em_std = all_embs.mean(), all_embs.std()\nem_size = all_embs.shape[1]\n\nword_index = token.word_index\nnb_words = min(max_features, len(word_index))\nem_matrix = np.random.normal(em_mean, em_std, (nb_words, em_size))\n# loop for every word\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    em_v = embedding_index.get(word)\n    if em_v is not None:\n        em_matrix[i] = em_v\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4391eebf0412807aa96d3f112f74f4bb9734e18d"},"cell_type":"code","source":"model = attention_model(em_matrix=em_matrix)\nmodel.fit(xtrain, ytrain, epochs=2, batch_size=512, validation_data=(xvalidate, yvaliate), verbose=1)\n\npred_vali_para = model.predict(xvalidate, batch_size=1024)\nbest_thre = evaluate(yvaliate, pred_vali_para)\n\npred_test_para = model.predict(xtest)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"626d150d35f17e49844283a99f5e44356ee9a478"},"cell_type":"code","source":"del embedding_index, all_embs, word_index, em_matrix\ngc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2be59874d4a5ae6589a86e3da613f6691d31b060"},"cell_type":"markdown","source":"#### Here I use a Linear Regression model to fit on this three model prediction on validation datasets to get a proper weights of different model."},{"metadata":{"trusted":true,"_uuid":"e3e1a16028e38bb5637b643272ad15c3e2ee1fd9"},"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nlr = LinearRegression()\n\nmodel_num = 2\n\npred_data = np.empty([len(pred_vali_para), model_num])\npred_data[:, 0] = pred_vali_glove.reshape(-1, )\n#pred_data[:, 1] = pred_vali_wiki.reshape(-1, )\npred_data[:, 1] = pred_vali_para.reshape(-1, )\n\nlr.fit(pred_data, yvaliate)\n\nweights = lr.coef_\n\nsub_pred_weighted = np.sum([pred_data[:, i]*weights[i] for i in range(model_num)], axis=0)\nbest_thre = evaluate(yvaliate ,sub_pred_weighted)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b56ce057e92b63e612d8ce46526f6ffc1b35912e"},"cell_type":"code","source":"# GET different model prediction result on test datasets\nsub_data = np.empty([len(xtest), model_num])\nsub_data[:, 0] = pred_test_glove.reshape(-1, )\n# sub_data[:, 1] = pred_test_wiki.reshape(-1, )\nsub_data[:, 1] = pred_test_para.reshape(-1, )","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5a5f277679ada9f93393cff41c7c39f845af753"},"cell_type":"markdown","source":"#### Submit result."},{"metadata":{"trusted":true,"_uuid":"3d10e5e0c493ed774c1aa21889038059c50f4d2a"},"cell_type":"code","source":"\n#sub_pred = 0.1 * pred_lstm_glove + 0.2*pred_bidi_lstm_glove + 0.1*pred_lstm_wiki + 0.2*pred_bidi_lstm_wiki+0.1*pred_lstm_para+0.3*pred_lstm_para\n# According to Linear Regression model result with different weights multiply with prediction.\nsub_pred = np.sum([sub_data[:, i]*weights[i] for i in range(model_num)], axis=0)\nsub_pred = (sub_pred > best_thre).astype(int)\n\nsub_df = pd.DataFrame({'qid':test.qid.values})\nsub_df['prediction'] = sub_pred\nsub_df.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}