{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport os\nfrom sklearn.model_selection import train_test_split\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\ntest_df = pd.read_csv('../input/test.csv')\nprint('Shape of train data',train_df.shape)\nprint('Shape of test data',test_df.shape)\nprint(train_df.columns)\nprint(test_df.columns)\nprint('Value counts in train target',train_df['target'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c3c82ad59f60d7eb0021b988c80149fcd784260"},"cell_type":"code","source":"train = train_df.fillna('_##_').values\nsubmission = test_df.fillna('_##_').values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"321f2b0357aaf6d125c48af7524e7054b5dd0458"},"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(train[:,1],train[:,2],test_size = 0.08)\nx_sub = submission[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65025d52cd8da008475b24e29ba963d3d7c92d9b"},"cell_type":"code","source":"EMBED_SIZE = 300\nNUM_WORDS = 100000\nMAXLEN = 70\n\ntokenizer = Tokenizer(num_words=NUM_WORDS)\ntokenizer.fit_on_texts(list(x_train))\n\ndef preprocess(text):\n    array = tokenizer.texts_to_sequences(text)\n    array = pad_sequences(array,maxlen = MAXLEN)\n    return array","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0baa01cc3833092f73a4c9c9f0f2f775bf3fd7a3"},"cell_type":"code","source":"x_train, x_test, x_sub = (preprocess(array) for array in [x_train, x_test, x_sub])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ffd4cad265e8e82645c3e9f945b9009caf21a16"},"cell_type":"code","source":"np.random.seed(2019)\nnp.random.shuffle(x_train)\nnp.random.shuffle(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"264d3d481a6a4bd4527ddfca63e4fa66cdd2de45"},"cell_type":"code","source":"print([tokenizer.index_word[i] if i != 0 else '<PAD>' for i in x_train[1,:]])\nprint([tokenizer.index_word[i]for i in x_train[1,:] if i != 0 ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1be482956eabe15b4497ff729c637d0b64a1962b"},"cell_type":"code","source":"for directory in os.listdir('../input/embeddings/'):\n    print('\\n../input/embeddings/{}: '.format(directory),os.listdir('../input/embeddings/{}'.format(directory)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d455a78fff891a0763a337cec0d4d7d2281195ee"},"cell_type":"code","source":"EMBEDDING_PATH = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr):\n    return word,np.asarray(arr, dtype='float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aae222a1d486e6ad9130944d5da13f1b804e2ca2"},"cell_type":"code","source":"embedding_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_PATH))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"12f9b97657525c86e6faa1e7d89d57d0f2987152"},"cell_type":"code","source":"all_embed = np.stack(embedding_index.values())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d3e46261cd50967a3945d35ba2f645016c11897"},"cell_type":"code","source":"embed_size = all_embed.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c140345b21c1d439c911254736e9ee7c70b94dad"},"cell_type":"code","source":"word_idx = tokenizer.word_index #dict 212124","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c7b6ce4936db5836a7f04345a5351c446b07970"},"cell_type":"code","source":"n_words = min(NUM_WORDS,len(word_idx)) # min 100000, 212124","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22f4b24e6f3acdee8e9170fdcc2bef5769e29714"},"cell_type":"code","source":"embedding_matrix = np.zeros((n_words,embed_size))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"915cbf2a436e4c5fbfcad533ff1a0c0fbc385707"},"cell_type":"code","source":"for word,idx in word_idx.items():\n    if idx >= n_words:\n        continue\n    else:\n        embedding_vector = embedding_index.get(word)\n    \n    if embedding_vector is not None:\n        embedding_matrix[idx] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ced5a1316d3b6fa01468c243d27e8ca0a9d0d774"},"cell_type":"code","source":"import random, os, sys\nimport numpy as np\nfrom keras.models import *\nfrom keras.layers import *\nfrom keras.callbacks import *\nfrom keras.initializers import *\nimport tensorflow as tf\nfrom keras.engine.topology import Layer\n\ntry:\n    from dataloader import TokenList, pad_to_longest\n    # for transformer\nexcept: pass\n\nclass LayerNormalization(Layer):\n    def __init__(self, eps=1e-6, **kwargs):\n        self.eps = eps\n        super(LayerNormalization, self).__init__(**kwargs)\n    def build(self, input_shape):\n        self.gamma = self.add_weight(name='gamma', shape=input_shape[-1:],\n                                     initializer=Ones(), trainable=True)\n        self.beta = self.add_weight(name='beta', shape=input_shape[-1:],\n                                    initializer=Zeros(), trainable=True)\n        super(LayerNormalization, self).build(input_shape)\n    def call(self, x):\n        mean = K.mean(x, axis=-1, keepdims=True)\n        std = K.std(x, axis=-1, keepdims=True)\n        return self.gamma * (x - mean) / (std + self.eps) + self.beta\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\nclass ScaledDotProductAttention():\n    def __init__(self, d_model, attn_dropout=0.1):\n        self.temper = np.sqrt(d_model)\n        self.dropout = Dropout(attn_dropout)\n    def __call__(self, q, k, v, mask):\n        attn = Lambda(lambda x:K.batch_dot(x[0],x[1],axes=[2,2])/self.temper)([q, k])\n        if mask is not None:\n            mmask = Lambda(lambda x:(-1e+10)*(1-x))(mask)\n            attn = Add()([attn, mmask])\n        attn = Activation('softmax')(attn)\n        attn = self.dropout(attn)\n        output = Lambda(lambda x:K.batch_dot(x[0], x[1]))([attn, v])\n        return output, attn\n\nclass MultiHeadAttention():\n    # mode 0 - big martixes, faster; mode 1 - more clear implementation\n    def __init__(self, n_head, d_model, d_k, d_v, dropout, mode=0, use_norm=True):\n        self.mode = mode\n        self.n_head = n_head\n        self.d_k = d_k\n        self.d_v = d_v\n        self.dropout = dropout\n        if mode == 0:\n            self.qs_layer = Dense(n_head*d_k, use_bias=False)\n            self.ks_layer = Dense(n_head*d_k, use_bias=False)\n            self.vs_layer = Dense(n_head*d_v, use_bias=False)\n        elif mode == 1:\n            self.qs_layers = []\n            self.ks_layers = []\n            self.vs_layers = []\n            for _ in range(n_head):\n                self.qs_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.ks_layers.append(TimeDistributed(Dense(d_k, use_bias=False)))\n                self.vs_layers.append(TimeDistributed(Dense(d_v, use_bias=False)))\n        self.attention = ScaledDotProductAttention(d_model)\n        self.layer_norm = LayerNormalization() if use_norm else None\n        self.w_o = TimeDistributed(Dense(d_model))\n\n    def __call__(self, q, k, v, mask=None):\n        d_k, d_v = self.d_k, self.d_v\n        n_head = self.n_head\n\n        if self.mode == 0:\n            qs = self.qs_layer(q)  # [batch_size, len_q, n_head*d_k]\n            ks = self.ks_layer(k)\n            vs = self.vs_layer(v)\n\n            def reshape1(x):\n                s = tf.shape(x)   # [batch_size, len_q, n_head * d_k]\n                x = tf.reshape(x, [s[0], s[1], n_head, d_k])\n                x = tf.transpose(x, [2, 0, 1, 3])  \n                x = tf.reshape(x, [-1, s[1], d_k])  # [n_head * batch_size, len_q, d_k]\n                return x\n            qs = Lambda(reshape1)(qs)\n            ks = Lambda(reshape1)(ks)\n            vs = Lambda(reshape1)(vs)\n\n            if mask is not None:\n                mask = Lambda(lambda x:K.repeat_elements(x, n_head, 0))(mask)\n            head, attn = self.attention(qs, ks, vs, mask=mask)  \n                \n            def reshape2(x):\n                s = tf.shape(x)   # [n_head * batch_size, len_v, d_v]\n                x = tf.reshape(x, [n_head, -1, s[1], s[2]]) \n                x = tf.transpose(x, [1, 2, 0, 3])\n                x = tf.reshape(x, [-1, s[1], n_head*d_v])  # [batch_size, len_v, n_head * d_v]\n                return x\n            head = Lambda(reshape2)(head)\n        elif self.mode == 1:\n            heads = []; attns = []\n            for i in range(n_head):\n                qs = self.qs_layers[i](q)   \n                ks = self.ks_layers[i](k) \n                vs = self.vs_layers[i](v) \n                head, attn = self.attention(qs, ks, vs, mask)\n                heads.append(head); attns.append(attn)\n            head = Concatenate()(heads) if n_head > 1 else heads[0]\n            attn = Concatenate()(attns) if n_head > 1 else attns[0]\n\n        outputs = self.w_o(head)\n        outputs = Dropout(self.dropout)(outputs)\n        if not self.layer_norm: return outputs, attn\n        outputs = Add()([outputs, q])\n        return self.layer_norm(outputs), attn\n\nclass PositionwiseFeedForward():\n    def __init__(self, d_hid, d_inner_hid, dropout=0.1):\n        self.w_1 = Conv1D(d_inner_hid, 1, activation='relu')\n        self.w_2 = Conv1D(d_hid, 1)\n        self.layer_norm = LayerNormalization()\n        self.dropout = Dropout(dropout)\n    def __call__(self, x):\n        output = self.w_1(x) \n        output = self.w_2(output)\n        output = self.dropout(output)\n        output = Add()([output, x])\n        return self.layer_norm(output)\n\nclass EncoderLayer():\n    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, dropout=0.1):\n        self.self_att_layer = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.pos_ffn_layer  = PositionwiseFeedForward(d_model, d_inner_hid, dropout=dropout)\n    def __call__(self, enc_input, mask=None):\n        output, slf_attn = self.self_att_layer(enc_input, enc_input, enc_input, mask=mask)\n        output = self.pos_ffn_layer(output)\n        return output, slf_attn\n\nclass DecoderLayer():\n    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, dropout=0.1):\n        self.self_att_layer = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.enc_att_layer  = MultiHeadAttention(n_head, d_model, d_k, d_v, dropout=dropout)\n        self.pos_ffn_layer  = PositionwiseFeedForward(d_model, d_inner_hid, dropout=dropout)\n    def __call__(self, dec_input, enc_output, self_mask=None, enc_mask=None):\n        output, slf_attn = self.self_att_layer(dec_input, dec_input, dec_input, mask=self_mask)\n        output, enc_attn = self.enc_att_layer(output, enc_output, enc_output, mask=enc_mask)\n        output = self.pos_ffn_layer(output)\n        return output, slf_attn, enc_attn\n\ndef GetPosEncodingMatrix(max_len, d_emb):\n    pos_enc = np.array([\n        [pos / np.power(10000, 2 * (j // 2) / d_emb) for j in range(d_emb)] \n        if pos != 0 else np.zeros(d_emb) \n            for pos in range(max_len)\n            ])\n    pos_enc[1:, 0::2] = np.sin(pos_enc[1:, 0::2]) # dim 2i\n    pos_enc[1:, 1::2] = np.cos(pos_enc[1:, 1::2]) # dim 2i+1\n    return pos_enc\n\ndef GetPadMask(q, k):\n    ones = K.expand_dims(K.ones_like(q, 'float32'), -1)\n    mask = K.cast(K.expand_dims(K.not_equal(k, 0), 1), 'float32')\n    mask = K.batch_dot(ones, mask, axes=[2,1])\n    return mask\n\ndef GetSubMask(s):\n    len_s = tf.shape(s)[1]\n    bs = tf.shape(s)[:1]\n    mask = K.cumsum(tf.eye(len_s, batch_shape=bs), 1)\n    return mask\n\nclass Encoder():\n    def __init__(self, d_model, d_inner_hid, n_head, d_k, d_v, \\\n                layers=6, dropout=0.1, word_emb=None, pos_emb=None):\n        self.emb_layer = word_emb\n        self.pos_layer = pos_emb\n        self.emb_dropout = Dropout(dropout)\n        self.layers = [EncoderLayer(d_model, d_inner_hid, n_head, d_k, d_v, dropout) for _ in range(layers)]\n        \n    def __call__(self, src_seq, src_pos, return_att=False, active_layers=999):\n        x = self.emb_layer(src_seq)\n        if src_pos is not None:\n            pos = self.pos_layer(src_pos)\n            x = Add()([x, pos])\n        x = self.emb_dropout(x)\n        if return_att: atts = []\n        mask = Lambda(lambda x:GetPadMask(x, x))(src_seq)\n        for enc_layer in self.layers[:active_layers]:\n            x, att = enc_layer(x, mask)\n            if return_att: atts.append(att)\n        return (x, atts) if return_att else x\n\n\nclass Transformer():\n    def __init__(self, len_limit, d_model=embed_size, \\\n              d_inner_hid=512, n_head=10, d_k=64, d_v=64, layers=2, dropout=0.1, \\\n              share_word_emb=False, **kwargs):\n        self.name = 'Transformer'\n        self.len_limit = len_limit\n        self.src_loc_info = True\n        self.d_model = d_model\n        self.decode_model = None\n        d_emb = d_model\n\n        pos_emb = Embedding(len_limit, d_emb, trainable=False, \\\n                            weights=[GetPosEncodingMatrix(len_limit, d_emb)])\n\n        i_word_emb = Embedding(NUM_WORDS, d_emb, weights=[embedding_matrix]) # Add Kaggle provided embedding here\n\n        self.encoder = Encoder(d_model, d_inner_hid, n_head, d_k, d_v, layers, dropout, \\\n                               word_emb=i_word_emb, pos_emb=pos_emb)\n\n        \n    def get_pos_seq(self, x):\n        mask = K.cast(K.not_equal(x, 0), 'int32')\n        pos = K.cumsum(K.ones_like(x, 'int32'), 1)\n        return pos * mask\n\n    def compile(self, active_layers=999):\n        src_seq_input = Input(shape=(None,))\n        src_seq = src_seq_input\n        src_pos = Lambda(self.get_pos_seq)(src_seq)\n        if not self.src_loc_info: src_pos = None\n\n        x = self.encoder(src_seq, src_pos, active_layers=active_layers)\n        # x = GlobalMaxPool1D()(x) # Not sure about this layer. Just wanted to reduce dimension\n        x = GlobalAveragePooling1D()(x)\n        outp = Dense(1, activation=\"sigmoid\")(x)\n\n        self.model = Model(inputs=src_seq_input, outputs=outp)\n        self.model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85c719840e4fbbbc378982d03f724a2ab5b9d659"},"cell_type":"code","source":"s2s = Transformer(MAXLEN, layers=1)\ns2s.compile()\nmodel = s2s.model\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcf2ddb02cd15119bf42e06a3203420c1074a4ac"},"cell_type":"code","source":"model.fit(x_train, y_train,batch_size=1024,epochs=1)\n# model.fit(x_train, y_train,batch_size=1024,epochs=1, validation_data=(x_test, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8400574351912022e8754af03acecf9b2d72ba17"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe56586fcbc417924cc63fc30ade82983a4459ca"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d4e51fbfca1df995d24515c8e2fd7a93648f75f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6b879336a45158ac6d8919ed5311a8e652bf27b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}