{"cells":[{"metadata":{"_uuid":"4a6e1070d4f6b5ef0dcd56cd3da5e98c52fe12cd"},"cell_type":"markdown","source":"Using multi-head self attention for classification\nThe implementation is from https://github.com/bojone/attention/blob/master/attention_keras.py\n\nMulti-head self attention is used to capture the relationship for each pair of words within a sentence.  Multi-head map the input many times to capture richer information\n\nSuppose the number of heads is nb_heads and the attention representation dimension is size_per_head, the total length of a sentence is time_step. For each head we calculate each pair of words similarity, it will give us [nb_heads, time_step, time_step] weights. Then we add the weights to each words and will get [time_step, size_per_head] representation for each head of a word. Finally we concat all heads representations and get [time_step, nb_heads*size_per_head]\n\n**However , the result is much worse than text classification attention LSTM model**, I think the reason may be the representation we get here only capture the inner structure of a sentence. Meanwhile,  text classification attention LSTM model directly capture which part of a sentence is imortant to classification. We can add some other structure after self attention  to make this model adapted to text classification."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\n%matplotlib inline\n\nfrom nltk.tokenize import TweetTokenizer\nimport datetime\nfrom scipy import stats\nfrom scipy.sparse import hstack, csr_matrix\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom wordcloud import WordCloud\nfrom collections import Counter\nfrom nltk.corpus import stopwords\nfrom nltk.util import ngrams\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import LinearSVC\nfrom sklearn.multiclass import OneVsRestClassifier\npd.set_option('max_colwidth',400)\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D, GRU, CuDNNGRU, CuDNNLSTM, BatchNormalization\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, MaxPooling1D, Add, Flatten\nfrom keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D\nfrom keras.models import Model, load_model\nfrom keras import initializers, regularizers, constraints, optimizers, layers, callbacks\nfrom keras import backend as K\nfrom keras.engine import InputSpec, Layer\nfrom keras.optimizers import Adam\n\nfrom keras.callbacks import ModelCheckpoint, TensorBoard, Callback, EarlyStopping\nfrom sklearn.preprocessing import OneHotEncoder\nfrom tqdm import tqdm\nimport re\n\nimport nltk\nfrom sklearn.utils import shuffle\nfrom tqdm import tqdm\n#nltk.download('stopwords')\nfrom sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18dce3a486e5b8f401838e88958067de979e4973"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.08, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 95000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_##_\").values\nval_X = val_df[\"question_text\"].fillna(\"_##_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values\n\n#shuffling the data\nnp.random.seed(2018)\ntrn_idx = np.random.permutation(len(train_X))\nval_idx = np.random.permutation(len(val_X))\n\ntrain_X = train_X[trn_idx]\nval_X = val_X[val_idx]\ntrain_y = train_y[trn_idx]\nval_y = val_y[val_idx]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47170893549ef2fb288381360fdbcfddef2f7f5c"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f20ece80e25b36101da743ff12fea7837efec21b"},"cell_type":"code","source":"class Position_Embedding(Layer):\n    \n    def __init__(self, size=None, mode='sum', **kwargs):\n        self.size = size\n        self.mode = mode\n        super(Position_Embedding, self).__init__(**kwargs)\n        \n    def call(self, x):\n        if (self.size == None) or (self.mode == 'sum'):\n            self.size = int(x.shape[-1])\n        batch_size,seq_len = K.shape(x)[0],K.shape(x)[1]\n        position_j = 1. / K.pow(10000., \\\n                                 2 * K.arange(self.size / 2, dtype='float32' \\\n                               ) / self.size)\n        position_j = K.expand_dims(position_j, 0)\n        position_i = K.cumsum(K.ones_like(x[:,:,0]), 1)-1 \n        position_i = K.expand_dims(position_i, 2)\n        position_ij = K.dot(position_i, position_j)\n        position_ij = K.concatenate([K.cos(position_ij), K.sin(position_ij)], 2)\n        if self.mode == 'sum':\n            return position_ij + x\n        elif self.mode == 'concat':\n            return K.concatenate([position_ij, x], 2)\n        \n    def compute_output_shape(self, input_shape):\n        if self.mode == 'sum':\n            return input_shape\n        elif self.mode == 'concat':\n            return (input_shape[0], input_shape[1], input_shape[2]+self.size)\n\n\n'''\noutput dimention: [batch_size, time_step, nb_head*size_per_head]\nevery word can be represented as a vector [nb_head*size_per_head]\n'''\nclass Attention(Layer):\n\n    def __init__(self, nb_head, size_per_head, **kwargs):\n        self.nb_head = nb_head\n        self.size_per_head = size_per_head\n        self.output_dim = nb_head*size_per_head\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.WQ = self.add_weight(name='WQ', \n                                  shape=(input_shape[0][-1], self.output_dim),\n                                  initializer='glorot_uniform',\n                                  trainable=True)\n        self.WK = self.add_weight(name='WK', \n                                  shape=(input_shape[1][-1], self.output_dim),\n                                  initializer='glorot_uniform',\n                                  trainable=True)\n        self.WV = self.add_weight(name='WV', \n                                  shape=(input_shape[2][-1], self.output_dim),\n                                  initializer='glorot_uniform',\n                                  trainable=True)\n        super(Attention, self).build(input_shape)\n        \n    def Mask(self, inputs, seq_len, mode='mul'):\n        if seq_len == None:\n            return inputs\n        else:\n            mask = K.one_hot(seq_len[:,0], K.shape(inputs)[1])\n            mask = 1 - K.cumsum(mask, 1)\n            for _ in range(len(inputs.shape)-2):\n                mask = K.expand_dims(mask, 2)\n            if mode == 'mul':\n                return inputs * mask\n            if mode == 'add':\n                return inputs - (1 - mask) * 1e12\n                \n    def call(self, x):\n        if len(x) == 3:\n            Q_seq,K_seq,V_seq = x\n            Q_len,V_len = None,None\n        elif len(x) == 5:\n            Q_seq,K_seq,V_seq,Q_len,V_len = x\n        Q_seq = K.dot(Q_seq, self.WQ)\n        Q_seq = K.reshape(Q_seq, (-1, K.shape(Q_seq)[1], self.nb_head, self.size_per_head))\n        Q_seq = K.permute_dimensions(Q_seq, (0,2,1,3))\n        K_seq = K.dot(K_seq, self.WK)\n        K_seq = K.reshape(K_seq, (-1, K.shape(K_seq)[1], self.nb_head, self.size_per_head))\n        K_seq = K.permute_dimensions(K_seq, (0,2,1,3))\n        V_seq = K.dot(V_seq, self.WV)\n        V_seq = K.reshape(V_seq, (-1, K.shape(V_seq)[1], self.nb_head, self.size_per_head))\n        V_seq = K.permute_dimensions(V_seq, (0,2,1,3))\n        A = K.batch_dot(Q_seq, K_seq, axes=[3,3]) / self.size_per_head**0.5\n        A = K.permute_dimensions(A, (0,3,2,1))\n        A = self.Mask(A, V_len, 'add')\n        A = K.permute_dimensions(A, (0,3,2,1))    \n        A = K.softmax(A)\n        O_seq = K.batch_dot(A, V_seq, axes=[3,2])\n        O_seq = K.permute_dimensions(O_seq, (0,2,1,3))\n        O_seq = K.reshape(O_seq, (-1, K.shape(O_seq)[1], self.output_dim))\n        O_seq = self.Mask(O_seq, Q_len, 'mul')\n        return O_seq\n        \n    def compute_output_shape(self, input_shape):\n        return (input_shape[0][0], input_shape[0][1], self.output_dim)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5a4507071b64d06d558d1f233e3dcb69f7c9d57d"},"cell_type":"code","source":"config = {\n    \"trainable\": False,\n    \"max_len\": 70,\n    \"max_features\": 95000,\n    \"embed_size\": 300,\n    \"units\": 64,\n    \"num_heads\": 8,\n    \"dr\": 0.5,\n    \"epochs\": 2,\n    \"model_checkpoint_path\": \"best_weights\",\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db7f2b28d018c203ec13153a52850e7d5e752bac"},"cell_type":"code","source":"def build_model(config):\n    inp = Input(shape = (config[\"max_len\"],))\n    \n    x = Embedding(config[\"max_features\"], config[\"embed_size\"], weights = [embedding_matrix], trainable = config[\"trainable\"])(inp)\n    x = Position_Embedding()(x)\n    x = Attention(config[\"num_heads\"], config[\"units\"])([x, x, x])  #output: [batch_size, time_step, nb_head*size_per_head]\n    x = GlobalAveragePooling1D()(x)\n    x = Dropout(config[\"dr\"])(x)\n    \n    x = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs = inp, outputs = x)\n    model.compile(\n        loss = \"binary_crossentropy\", \n        #optimizer = Adam(lr = config[\"lr\"], decay = config[\"lr_d\"]), \n        optimizer = \"nadam\",\n        metrics = [\"accuracy\"])\n    \n    return model\n\nmodel = build_model(config)\nmodel.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dee5a990edfa9c89e7bb9df9e0db6bb07815a5a7"},"cell_type":"code","source":"pred_cnn_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_cnn_val_y>thresh).astype(int))))\n\npred_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e974c6f3a7b215e73ee5a05d4453558834a9cc3"},"cell_type":"code","source":"pred_test_y = (pred_test_y > 0.11).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b51290cdf402a8cb9443c3690e0e23ce92e20cad"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}