{"cells":[{"metadata":{"_uuid":"565efe44ca706182eca35d63c14eda622e966b43"},"cell_type":"markdown","source":"Some thoughts:\n\n* Is attention all you need? That may be true for seq2seq learning, but not in text summarization\n\n* Since other Kagglers found adding attention to LSTM helps, why not add the multi head attention to LSTM?\n\n* Blending rocks (multi head attention)\n\n\nReference:\n\n* Attention Is All You Need: https://arxiv.org/abs/1706.03762\n\n* Transformer in Keras: https://github.com/Lsdefine/attention-is-all-you-need-keras/blob/master/transformer.py\n    \n* SRK: https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings . There is not much changed from this kernel except the nueral net architecture and final weights of the embeddings.\n\n* Attention model from Khoi Ngyuen: https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb\n\n* https://www.kaggle.com/shujian/different-embeddings-with-attention-fork-fork"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Flatten, GlobalAveragePooling1D, Reshape\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7612d1a3d79754ca6a48cc21d42391b1917255bf"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"72530a51393c9d3697b5127be319aff3ddd860f7"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 95000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_##_\").values\nval_X = val_df[\"question_text\"].fillna(\"_##_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4c8211ccd5672607b26ec7d6470f7ab635e0334"},"cell_type":"code","source":"#shuffling the data\nnp.random.seed(2018)\ntrn_idx = np.random.permutation(len(train_X))\nval_idx = np.random.permutation(len(val_X))\n\ntrain_X = train_X[trn_idx]\nval_X = val_X[val_idx]\ntrain_y = train_y[trn_idx]\nval_y = val_y[val_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7c5ad7277ad7f33df9fbdd3243d68b9d7260e94"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a588d1ab2113f12f2ef1937f0a6f6ad9ca3b802d"},"cell_type":"code","source":"import random, os, sys\nimport numpy as np\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.initializers import *\nimport tensorflow as tf\nfrom keras.engine.topology import Layer\n\ntry:\n    from dataloader import TokenList, pad_to_longest\n    # for transformer\nexcept: pass\n\nclass LayerNormalization(Layer):\n    def __init__(self, eps=1e-6, **kwargs):\n        self.eps = eps\n        super(LayerNormalization, self).__init__(**kwargs)\n    def build(self, input_shape):\n        self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],\n                                     initializer=Ones(), trainable=True)\n        self.beta = self.add_weight(name='beta', shape=input_shape[-1:],\n                                    initializer=Zeros(), trainable=True)\n        super(LayerNormalization, self).build(input_shape)\n    def call(self, x):\n        mean = K.mean(x, axis=-1, keepdims=True)\n        std = K.std(x, axis=-1, keepdims=True)\n        return self.gamma * (x - mean) / (std + self.eps) + self.beta\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\nclass ScaledDotProductAttention():\n    def __init__(self, d_model, attn_dropout=0.1):\n        self.temper = np.sqrt(d_model)\n        self.dropout = Dropout(attn_dropout)\n    def __call__(self, q, k, v, mask):\n        attn = Lambda(lambda x:K.batch_dot(x[0],x[1],axes=[2,2])/self.temper)([q, k])\n        if mask is not None:\n            mmask = Lambda(lambda x:(-1e+10)*(1-x))(mask)\n            attn = Add()([attn, mmask])\n        attn = Activation('softmax')(attn)\n        attn = self.dropout(attn)\n        output = Lambda(lambda x:K.batch_dot(x[0], x[1]))([attn, v])\n        return output, attn\n\nclass MultiHeadAttention():\n    # mode 0 - big martixes, faster; mode 1 - more clear implementation\n    def __init__(self, n_head, d_model, d_k, d_v, dropout, mode=0, use_norm=True):\n        self.mode = mode\n        self.n_head = n_head\n        self.d_k = d_k\n        self.d_v = d_v\n        self.dropout = dropout\n        if mode == 0:\n            self.qs_layer = Dense(n_head*d_k, use_bias=False)\n            self.ks_layer = Dense(n_head*d_k, use_bias=False)\n            self.vs_layer = Dense(n_head*d_v, use_bias=False)\n        elif mode == 1:\n            self.qs_layers = []\n            self.ks_layers = []\n            self.vs_layers = []\n            for _ in range(n_head):\n                self.qs_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.ks_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.vs_layers.append(TimeDistributed(Dense(d_v, use_bias=False)))\n        self.attention = ScaledDotProductAttention(d_model)\n        self.layer_norm = LayerNormalization() if use_norm else None\n        self.w_o = TimeDistributed(Dense(d_model))\n\n    def __call__(self, q, k, v, mask=None):\n        d_k, d_v = self.d_k, self.d_v\n        n_head = self.n_head\n\n        if self.mode == 0:\n            qs = self.qs_layer(q)  # [batch_size, len_q, n_head*d_k]\n            ks = self.ks_layer(k)\n            vs = self.vs_layer(v)\n\n            def reshape1(x):\n                s = tf.shape(x)   # [batch_size, len_q, n_head * d_k]\n                x = tf.reshape(x, [s[0], s[1], n_head, d_k])\n                x = tf.transpose(x, [2, 0, 1, 3])  \n                x = tf.reshape(x, [-1, s[1], d_k])  # [n_head * batch_size, len_q, d_k]\n                return x\n            qs = Lambda(reshape1)(qs)\n            ks = Lambda(reshape1)(ks)\n            vs = Lambda(reshape1)(vs)\n\n            if mask is not None:\n                mask = Lambda(lambda x:K.repeat_elements(x, n_head, 0))(mask)\n            head, attn = self.attention(qs, ks, vs, mask=mask)  \n                \n            def reshape2(x):\n                s = tf.shape(x)   # [n_head * batch_size, len_v, d_v]\n                x = tf.reshape(x, [n_head, -1, s[1], s[2]]) \n                x = tf.transpose(x, [1, 2, 0, 3])\n                x = tf.reshape(x, [-1, s[1], n_head*d_v])  # [batch_size, len_v, n_head * d_v]\n                return x\n            head = Lambda(reshape2)(head)\n        elif self.mode == 1:\n            heads = []; attns = []\n            for i in range(n_head):\n                qs = self.qs_layers[i](q)   \n                ks = self.ks_layers[i](k) \n                vs = self.vs_layers[i](v) \n                head, attn = self.attention(qs, ks, vs, mask)\n                heads.append(head); attns.append(attn)\n            head = Concatenate()(heads) if n_head > 1 else heads[0]\n            attn = Concatenate()(attns) if n_head > 1 else attns[0]\n\n        outputs = self.w_o(head)\n        outputs = Dropout(self.dropout)(outputs)\n        if not self.layer_norm: return outputs, attn\n        # outputs = Add()([outputs, q]) # sl: fix\n        return self.layer_norm(outputs), attn\n\nclass PositionwiseFeedForward():\n    def __init__(self, d_hid, d_inner_hid, dropout=0.1):\n        self.w_1 = Conv1D(d_inner_hid, 1, activation='relu')\n        self.w_2 = Conv1D(d_hid, 1)\n        self.layer_norm = LayerNormalization()\n        self.dropout = Dropout(dropout)\n    def __call__(self, x):\n        output = self.w_1(x) \n        output = self.w_2(output)\n        output = self.dropout(output)\n        output = Add()([output, x])\n        return self.layer_norm(output)\n\nclass EncoderLayer():\n    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, dropout=0.1):\n        self.self_att_layer = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.pos_ffn_layer  = PositionwiseFeedForward(d_model, d_inner_hid, dropout=dropout)\n    def __call__(self, enc_input, mask=None):\n        output, slf_attn = self.self_att_layer(enc_input, enc_input, enc_input, mask=mask)\n        output = self.pos_ffn_layer(output)\n        return output, slf_attn\n\n\ndef GetPosEncodingMatrix(max_len, d_emb):\n    pos_enc = np.array([\n        [pos / np.power(10000, 2 * (j // 2) / d_emb) for j in range(d_emb)] \n        if pos != 0 else np.zeros(d_emb) \n            for pos in range(max_len)\n            ])\n    pos_enc[1:, 0::2] = np.sin(pos_enc[1:, 0::2]) # dim 2i\n    pos_enc[1:, 1::2] = np.cos(pos_enc[1:, 1::2]) # dim 2i+1\n    return pos_enc\n\ndef GetPadMask(q, k):\n    ones = K.expand_dims(K.ones_like(q, 'float32'), -1)\n    mask = K.cast(K.expand_dims(K.not_equal(k, 0), 1), 'float32')\n    mask = K.batch_dot(ones, mask, axes=[2,1])\n    return mask\n\ndef GetSubMask(s):\n    len_s = tf.shape(s)[1]\n    bs = tf.shape(s)[:1]\n    mask = K.cumsum(tf.eye(len_s, batch_shape=bs), 1)\n    return mask\n\nclass Transformer():\n    def __init__(self, len_limit, embedding_matrix, d_model=embed_size, \\\n              d_inner_hid=512, n_head=10, d_k=64, d_v=64, layers=2, dropout=0.1, \\\n              share_word_emb=False, **kwargs):\n        self.name = 'Transformer'\n        self.len_limit = len_limit\n        self.src_loc_info = False # True # sl: fix later\n        self.d_model = d_model\n        self.decode_model = None\n        d_emb = d_model\n\n        pos_emb = Embedding(len_limit, d_emb, trainable=False, \\\n                            weights=[GetPosEncodingMatrix(len_limit, d_emb)])\n\n        i_word_emb = Embedding(max_features, d_emb, weights=[embedding_matrix]) # Add Kaggle provided embedding here\n\n        self.encoder = Encoder(d_model, d_inner_hid, n_head, d_k, d_v, layers, dropout, \\\n                               word_emb=i_word_emb, pos_emb=pos_emb)\n\n        \n    def get_pos_seq(self, x):\n        mask = K.cast(K.not_equal(x, 0), 'int32')\n        pos = K.cumsum(K.ones_like(x, 'int32'), 1)\n        return pos * mask\n\n    def compile(self, active_layers=999):\n        src_seq_input = Input(shape=(None, ))\n        x = Embedding(max_features, embed_size, weights=[embedding_matrix])(src_seq_input)\n        \n        # LSTM before attention layers\n        x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n        x = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x) \n        \n        x, slf_attn = MultiHeadAttention(n_head=3, d_model=300, d_k=64, d_v=64, dropout=0.1)(x, x, x)\n        \n        avg_pool = GlobalAveragePooling1D()(x)\n        max_pool = GlobalMaxPooling1D()(x)\n        conc = concatenate([avg_pool, max_pool])\n        conc = Dense(64, activation=\"relu\")(conc)\n        x = Dense(1, activation=\"sigmoid\")(conc)   \n        \n        \n        self.model = Model(inputs=src_seq_input, outputs=x)\n        self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"370eb8437768b36204404e712692e48746b76476","scrolled":true},"cell_type":"code","source":"s2s = Transformer(64, embedding_matrix, layers=1)\ns2s.compile()\nmodel = s2s.model\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd4ba6a5810c9c4220c491d84b91557d32a649ca"},"cell_type":"code","source":"## Train the model \nmodel.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a1ae9ca76b82adb6c88824c35b9ccabb4bec612"},"cell_type":"code","source":"pred_val_y = model.predict([val_X], batch_size=1024, verbose=1)\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3130e796074bd8ecf6d35a6121593b4af5667f8e"},"cell_type":"code","source":"pred_test_y = model.predict([test_X], batch_size=1024, verbose=1)\n\npred_test_y = (pred_test_y > best_thresh).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}