{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fea4bcca9ac621a7bda6a592daf5e2e0ea0fc14d"},"cell_type":"code","source":"# Libraries\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom keras.preprocessing.text import Tokenizer\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.models import Sequential\nfrom keras.layers import Dense , LSTM , Embedding , Conv1D , Bidirectional , GRU , Dropout\nfrom keras.layers import GlobalMaxPool1D, Dropout, Activation,CuDNNLSTM\nfrom keras.layers import MaxPooling1D, BatchNormalization,Conv2D,Flatten","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10dad35e75f1c5af0faefec8fbed56ded216b925"},"cell_type":"markdown","source":"**Attention Class**"},{"metadata":{"trusted":true,"_uuid":"eebd98202f18046673f63bfe9bdce8016dec5321"},"cell_type":"code","source":"from keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab4c945eb37bc9b5466ca11c402c999667de1028"},"cell_type":"code","source":"# https://www.kaggle.com/qqgeogor/keras-lstm-attention-glove840b-lb-0-043\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# Training and Test set processing\ntrain = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\ntrain_data, validation_data = train_test_split(train, test_size=.1, random_state=1234)\nembed_size = 300 # how big is each word vector\nmax_features = 900000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\ntrain_X = train_data[\"question_text\"].fillna(\"_na_\").values\nval_X = validation_data[\"question_text\"].fillna(\"_na_\").values\ntest_X = test[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X)+list(val_X)+list(test_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X,maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_data['target'].values\nval_y = validation_data['target'].values\nprint(\"Done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8595a2d0e28f18f03acb6e63e8d25ddbec1c74f"},"cell_type":"code","source":"import gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4a21d96efe945ffda6b968784bda7185ea71cfd"},"cell_type":"code","source":"#Using Embeddings\nembedding_index = dict()\nf = open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt',encoding='utf8')\n\nfor line in f:\n    values = line.split(\" \")\n    words = values[0]\n    coefs = np.asarray(values[1:], dtype='float32')\n    embedding_index[words]= coefs\n    \nf.close()\nembedding_matrix_glove = np.zeros((max_features, 300))\nfor word, index in tokenizer.word_index.items():\n    if index > max_features - 1:\n        break\n    else:\n        embedding_vector = embedding_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix_glove[index] = embedding_vector\nprint('Null word embeddings: %d' % np.sum(np.sum(embedding_matrix_glove, axis=1) == 0))\ndel embedding_index; gc.collect()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"084224c7fccb4b4573ffa0182008c448b27ecb29"},"cell_type":"code","source":"from keras import backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c0b90f590c85d3c2c4d7f2430e2f4eba723438eb"},"cell_type":"code","source":"model=Sequential()\nmodel.add(Embedding(max_features, 300, weights=[embedding_matrix_glove],input_length=maxlen,trainable=False) )\nmodel.add(Bidirectional(CuDNNLSTM(128, return_sequences=True)))\nmodel.add(Bidirectional(CuDNNLSTM(64, return_sequences=True)))\nmodel.add(Attention(maxlen))\nmodel.add(Dense(32, activation=\"relu\"))\nmodel.add(Dense(1, activation=\"sigmoid\"))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(\"Done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10c6ded1ba78f17fbfb3f76dd74d0958478139c1"},"cell_type":"code","source":"#training\nmodel.fit(train_X,train_y,epochs=2,batch_size=512)\nprint(\"Done\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"89c0defc90faf0b3c36f25008240030dc00fc127"},"cell_type":"code","source":"from sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e3199fd2e1823d76132888daf3f6dc40dd3c1a4"},"cell_type":"code","source":"pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)\n\nbest_thresh = 0.5\nbest_score = 0.0\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    score = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n    if score > best_score:\n        best_thresh = thresh\n        best_score = score\n\nprint(\"Val F1 Score: {:.4f}\".format(best_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1db134d0693882bc1fa56b8ae1336eccacfe18a6"},"cell_type":"code","source":"best_thresh","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a56504dcedb4fa4e578ff5997feec915da87e49f"},"cell_type":"code","source":"pred_test_y = model.predict([test_X], batch_size=1024, verbose=1)\npred_test_y = (pred_test_y>0.41)                                    #changing the threshold in this version\nout_df = pd.DataFrame({\"qid\":test[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ded3d9365cc3ddc02970f19dec48bd8becc6d797"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}