{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"baed6500ef978404192514777b995175f0a828f8"},"cell_type":"code","source":"import itertools\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_curve, auc, confusion_matrix\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import (\n    Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, \n    Conv1D ,GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D,\n    Conv2D, MaxPool2D, concatenate,\n    Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D, Bidirectional, \n)\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras.engine.topology import Layer\nfrom keras import metrics\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras import backend as K\n\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10c2a8799b96863fd3334257e46043f9a2b44cff"},"cell_type":"code","source":"DEBUG = False\nEMBED_SIZE = 300\nMAX_FEATURES = 10000 if DEBUG else 90000\nSEQUENCE_LENGTH = 50\nGLOVE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nWIKI = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\nPARAGRAM = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7eda70eb76d1d133edb289ba5a6f081ecaf9d348"},"cell_type":"markdown","source":" # Load  data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\ntest_df = pd.read_csv('../input/test.csv')\nif DEBUG:\n    train_df = train_df[:100000]\ntrain_df['question_text'] = train_df['question_text'].str.lower()\ntest_df['question_text'] = test_df['question_text'].str.lower()\nDEBUG, train_df.shape, test_df.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"013835c7e0fb5e17acec4d475e11279df123a171"},"cell_type":"markdown","source":"# Embedding"},{"metadata":{"trusted":true,"_uuid":"55887f2aeac053b79da64d65cfc939dbae9453f7"},"cell_type":"code","source":"def load_embedding(file, tokenizer):\n    def split(line):\n        word, *arr = line.split(' ')\n        return word, np.asarray(arr, dtype='float32')\n    with open(file, encoding='utf8', errors='ignore') as f:\n        if DEBUG:\n            embeddings = dict(split(line) for line in tqdm(itertools.islice(f, 10000)) if len(line) > 100)\n        else:\n            embeddings = dict(split(line) for line in tqdm(f) if len(line) > 100)\n    values = np.stack(embeddings.values())\n    mean = values.mean()\n    std = values.std()\n    n_words = min(MAX_FEATURES, len(tokenizer.word_index))\n    \n    embedding_matrix = np.random.normal(mean, std, (n_words, EMBED_SIZE))\n    for word, i in tokenizer.word_index.items():\n        if i < MAX_FEATURES and word in embeddings:\n            embedding_matrix[i] = embeddings[word]\n            \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbfe774b1fb44e5c563d2b07e00291a9cb3296f7"},"cell_type":"markdown","source":"# Models"},{"metadata":{"trusted":true,"_uuid":"b98f7d79afac0bbfc8977e4a3fbbf47cbbeec00e"},"cell_type":"code","source":"# https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb\n# https://www.kaggle.com/qqgeogor/keras-lstm-attention-glove840b-lb-0-043\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n        if self.bias:\n            eij += self.b\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n        \n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b555627f50bce1788fe1d6672bb827d386a26e6a"},"cell_type":"code","source":"def model_gru_atten_3(embedding):\n    input_ = Input(shape=(SEQUENCE_LENGTH,))\n    x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding])(input_)\n    x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(96, return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n    x = Attention(SEQUENCE_LENGTH)(x)\n    x = Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=input_, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"134d088c42f59504ef8bb55af1f678ba5260ba50"},"cell_type":"code","source":"class AveragedModel:\n    def __init__(self, tokenizer):\n        self.embeddings = [\n            load_embedding(GLOVE, tokenizer),\n            load_embedding(WIKI, tokenizer),\n            load_embedding(PARAGRAM, tokenizer),\n        ]\n        self.models = [model_gru_atten_3(embedding) for embedding in self.embeddings]\n        \n    def fit(self, *args, **kwargs):\n        for model in self.models:\n            model.fit(*args, **kwargs)\n            \n    def predict(self, *args, **kwargs):\n        outputs = [model.predict(*args, **kwargs) for model in self.models]\n        return np.array(outputs).mean(axis=0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2667302889b8bf2068b00c0cf31bda9e1ece5998"},"cell_type":"markdown","source":"# Cross Validation"},{"metadata":{"trusted":true,"_uuid":"c7a3ea4d89fe6ce2dd5f3b6889d94cf5d591d93c"},"cell_type":"code","source":"# X = train_df[\"question_text\"]\n# y = train_df['target'].values\n# predicted_y = np.zeros_like(y, dtype='float32')\n\n# indices = []\n# splitted_Xy = []\n# models = []\n\n# splitter = StratifiedKFold(2, random_state=0)\n\n\n# for train_index, test_index in splitter.split(X, y):\n#     indices.append((train_index, test_index))\n    \n#     tokenizer = Tokenizer(num_words=MAX_FEATURES)\n#     tokenizer.fit_on_texts(X[train_index])\n    \n#     train_X = tokenizer.texts_to_sequences(X[train_index])\n#     train_X = pad_sequences(train_X, maxlen=SEQUENCE_LENGTH)\n#     test_X = tokenizer.texts_to_sequences(X[test_index])\n#     test_X = pad_sequences(test_X, maxlen=SEQUENCE_LENGTH)\n#     train_y = y[train_index]\n#     test_y = y[test_index]\n\n#     splitted_Xy.append((train_X, test_X, train_y, test_y))\n#     model = AveragedModel(tokenizer)\n#     models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"630aafe49d925c518296598d72fe92d80fa97703"},"cell_type":"code","source":"# for model, (train_X, test_X, train_y, test_y), (train_index, test_index) in zip(models, splitted_Xy, indices):\n#     model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(test_X, test_y), verbose=0)\n    \n#     yp = model.predict([test_X], batch_size=512, verbose=0).flatten()\n#     predicted_y[test_index] = yp","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc768d15c0c2203eb7f4fe7a8ee94df4d3fec099"},"cell_type":"code","source":"# def plot_base():\n#     plt.plot([0, 1], [0, 1], color='navy', linestyle='--')\n#     plt.xlim([0.0, 1.0])\n#     plt.ylim([0.0, 1.05])\n#     plt.xlabel('False Positive Rate')\n#     plt.ylabel('True Positive Rate')\n#     plt.title('Receiver operating characteristic example')\n#     plt.legend(loc=\"lower right\")\n#     plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40565b865ef31b9ac2e7f47441814132a307764a"},"cell_type":"code","source":"# fpr, tpr, thresholds = roc_curve(y, predicted_y, pos_label=1)\n# roc_auc = auc(fpr, tpr)\n# plt.plot(fpr, tpr, color='darkorange', label='ROC curve (area = %0.3f)' % roc_auc)\n    \n# plot_base()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23b93f479c3e8ea24acb9ab3317481dd9e0d1937"},"cell_type":"code","source":"# threshold = 0.5\n# confusion_matrix(y, (predicted_y > threshold).astype(int))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb1d7ae06fb4c92f82ac77372f03a7cb8cd682c6"},"cell_type":"code","source":"# f1_score(y, predicted_y > threshold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"967e9a3a34b28b51e3ad7704fa236a18037ad588"},"cell_type":"code","source":"# Save for later plotting\n# def find_best_threshold(y, predicted_y):\n#     best_f1, best_threshold = 0, 0\n#     for threshold in np.arange(0.01, 0.99, 0.01):\n#         f1 = f1_score(y, predicted_y > threshold)\n#         if f1 > best_f1:\n#             best_f1, best_threshold = f1, threshold\n#     return best_f1, best_threshold\n\n# best_f1, best_threshold = find_best_threshold(y, predicted_y)\n# best_f1, best_threshold","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f597329f27de23a69c11df719483d120c0a99192"},"cell_type":"markdown","source":"# Train and predict"},{"metadata":{"trusted":true,"_uuid":"8118488b36fbdffcf3af4b0ccba3331c6a07281c"},"cell_type":"code","source":"# tokenizer = Tokenizer(num_words=MAX_FEATURES)\n# tokenizer.fit_on_texts(X)\n\n# train_X = tokenizer.texts_to_sequences(X)\n# train_X = pad_sequences(train_X, maxlen=SEQUENCE_LENGTH)\n# train_y = y\n\n# model = AveragedModel(tokenizer)\n# model.fit(train_X, train_y, batch_size=512, epochs=1, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36f2bf53c81ad8ac249b1d07a1aeb287a342f192"},"cell_type":"code","source":"# test_X = tokenizer.texts_to_sequences(test_df[\"question_text\"])\n# test_X = pad_sequences(test_X, maxlen=SEQUENCE_LENGTH)\n\n# print('Tokenized test dataset')\n# prediction = model.predict([test_X], batch_size=1024, verbose=0).flatten()\n# predicted_label = (prediction > threshold).astype(int)\n# positive = predicted_label.mean()\n# print(f'Tested on {model}: {positive:.3%} positive')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"76007eb154a7306f83d34b57f243254c4889f5c1"},"cell_type":"code","source":"# out_df = pd.read_csv('../input/test.csv', usecols=['qid'])\n# out_df = pd.DataFrame({'qid': out_df['qid'].values})\n# out_df['prediction'] = predicted_label\n# out_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dfa485000f5e47419c9117efdf5f24caeae5c51b"},"cell_type":"markdown","source":"# Robustness Validation"},{"metadata":{"trusted":true,"_uuid":"665cc5736b1d53bd69897a2fa0ac2465b5d08a75"},"cell_type":"code","source":"# import random\n# def drop_randomly(sequences, ratio=.1):\n#     for sequence in sequences:\n#         if random.random() < ratio and sequence:\n#             del sequence[random.randint(0, len(sequence) - 1)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3d9bdb3018a4819db682d57f5664d75853323f06"},"cell_type":"code","source":"# X = train_df[\"question_text\"]\n# y = train_df['target'].values\n# predicted_y = np.zeros_like(y, dtype='float32')\n\n# indices = []\n# splitted_Xy = []\n# models = []\n\n# splitter = StratifiedKFold(2, random_state=0)\n\n\n# for train_index, test_index in splitter.split(X, y):\n#     indices.append((train_index, test_index))\n    \n#     tokenizer = Tokenizer(num_words=MAX_FEATURES)\n#     tokenizer.fit_on_texts(X[train_index])\n    \n#     train_X = tokenizer.texts_to_sequences(X[train_index])\n#     drop_randomly(train_X, ratio=.1) # test for robustness\n#     train_X = pad_sequences(train_X, maxlen=SEQUENCE_LENGTH)\n#     test_X = tokenizer.texts_to_sequences(X[test_index])\n#     test_X = pad_sequences(test_X, maxlen=SEQUENCE_LENGTH)\n#     train_y = y[train_index]\n#     test_y = y[test_index]\n\n#     splitted_Xy.append((train_X, test_X, train_y, test_y))\n#     model = AveragedModel(tokenizer)\n#     models.append(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e57f4ddb178f0adfaf54d82a4a49c5c6776404e3"},"cell_type":"code","source":"# for model, (train_X, test_X, train_y, test_y), (train_index, test_index) in zip(models, splitted_Xy, indices):\n#     model.fit(train_X, train_y, batch_size=512, epochs=1, verbose=0)\n    \n#     yp = model.predict([test_X], batch_size=1024, verbose=0).flatten()\n#     predicted_y[test_index] = yp\n    \n    \n# fpr, tpr, thresholds = roc_curve(y, predicted_y, pos_label=1)\n# roc_auc = auc(fpr, tpr)\n# plt.plot(fpr, tpr, color='darkorange', label='ROC curve (area = %0.3f)' % roc_auc)\n# plot_base()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2cb3774cc9f9bd06d3d2916025245d747d14c2a8"},"cell_type":"markdown","source":"# Refinement"},{"metadata":{"trusted":true,"_uuid":"d0aedb484d240ff01ad6829bca25b06a5f36af55"},"cell_type":"code","source":"# rocs = [(fpr, tpr, thresholds, roc_auc, best_f1, best_threshold)]\n# for i in range(2):\n#     for model, (train_X, test_X, train_y, test_y), (train_index, test_index) in zip(models, splitted_Xy, indices):\n#         model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(test_X, test_y), verbose=0)\n#         yp = model.predict([test_X], batch_size=1024, verbose=0).flatten()\n#         predicted_y[test_index] = yp\n        \n#     fpr_, tpr_, thresholds_ = roc_curve(y, predicted_y, pos_label=1)\n#     roc_auc_ = auc(fpr_, tpr_)\n    \n#     best_f1, best_threshold = find_best_threshold(y, predicted_y)\n#     rocs.append((fpr_, tpr_, thresholds_, roc_auc_, best_f1, best_threshold))\n#     print(f'Epoch {i + 2}: ROC_AUC={roc_auc_}')\n#     print('Confusion matrix', confusion_matrix(y, (predicted_y > best_threshold).astype(int)))\n    \n# rocs","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48c9dc0506a57e097a2fcfb84c27ccb5c9d19570"},"cell_type":"code","source":"# plt.figure(figsize=(10, 7))\n# for i, (fpr_, tpr_, thresholds_, roc_auc_, best_f1, best_threshold) in enumerate(rocs):\n#     plt.plot(fpr_, tpr_, label=f'{i + 1} epochs (area = {roc_auc_:.3f}, best f1={best_f1:.3f})')\n# plot_base()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c61e7630209c05499e6039c6762026aeac44605d"},"cell_type":"code","source":"# aucs = [i[3] for i in rocs]\n# argmax = np.argmax(aucs)\n# best_epochs = argmax + 1\n# best_threshold = rocs[argmax][5]\n# aucs, best_epochs, aucs[argmax], best_threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1f51826313d30295881c595581722aa3cb588ffc"},"cell_type":"code","source":"best_epochs = 2\nbest_threshold = 0.35\nX = train_df[\"question_text\"]\ny = train_df['target'].values\n\ntokenizer = Tokenizer(num_words=MAX_FEATURES)\ntokenizer.fit_on_texts(X)\n\ntrain_X = tokenizer.texts_to_sequences(X)\ntrain_X = pad_sequences(train_X, maxlen=SEQUENCE_LENGTH)\ntrain_y = y\n\nmodel = AveragedModel(tokenizer)\nmodel.fit(train_X, train_y, batch_size=512, epochs=best_epochs, verbose=0)\n\n\ntest_X = tokenizer.texts_to_sequences(test_df[\"question_text\"])\ntest_X = pad_sequences(test_X, maxlen=SEQUENCE_LENGTH)\n\nprint('Tokenized test dataset')\nprediction = model.predict([test_X], batch_size=1024, verbose=0).flatten()\npredicted_label = (prediction > best_threshold).astype(int)\npositive = predicted_label.mean()\nprint(f'Tested on {model}: {positive:.3%} positive')\n\nout_df = pd.read_csv('../input/test.csv', usecols=['qid'])\nout_df = pd.DataFrame({'qid': out_df['qid'].values})\nout_df['prediction'] = predicted_label\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9712a17670894061b2e00e03fee5894fdb09b047"},"cell_type":"code","source":"def visualize_layers():\n    plt.figure(figsize=(20, 8))\n    for i, sub_model in enumerate(model.models):\n        for j in range(3):\n            plt.subplot(3, 3, i * 3 + j + 1)\n            layer = sub_model.layers[j + 2]\n            weights = layer.backward_layer.get_weights()[1]\n            plt.imshow(weights, cmap='viridis', interpolation='nearest')\n            plt.title(f'Embedding {i + 1}, bidirection GRU layer {j + 1}')\nvisualize_layers()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}