{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\ntest_df = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e02b0e47e46bb5b9a904fed96182335eda8de59"},"cell_type":"code","source":"X_train = train_df[\"question_text\"].fillna(\"na\").values\nX_test = test_df[\"question_text\"].fillna(\"na\").values\ny = train_df[\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af1bd51a516911116691ed82122f23a83edc73da"},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense, Embedding, concatenate\nfrom keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D\nfrom keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras.optimizers import *\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nimport gc\nfrom sklearn import metrics\nfrom keras import regularizers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7029f9ecffebecab7efdc39c619674d4dff15b00"},"cell_type":"code","source":"maxlen = 50\nmax_features = 50000\nembed_size = 300\n\ntokenizer = text.Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train) + list(X_test))\n\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\n\nx_train = sequence.pad_sequences(X_train, maxlen=maxlen)\nx_test = sequence.pad_sequences(X_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58db1584ca04318065c825254b640517b474cdcc"},"cell_type":"code","source":"def attention_3d_block(inputs):\n    # inputs.shape = (batch_size, time_steps, input_dim)\n    TIME_STEPS = inputs.shape[1].value\n    SINGLE_ATTENTION_VECTOR = False\n    \n    input_dim = int(inputs.shape[2])\n    a = Permute((2, 1))(inputs)\n    a = Reshape((input_dim, TIME_STEPS))(a) # this line is not useful. It's just to know which dimension is what.\n    a = Dense(TIME_STEPS, activation='softmax')(a)\n    if SINGLE_ATTENTION_VECTOR:\n        a = Lambda(lambda x: K.mean(x, axis=1))(a)\n        a = RepeatVector(input_dim)(a)\n    a_probs = Permute((2, 1))(a)\n    output_attention_mul = Multiply()([inputs, a_probs])\n    return output_attention_mul","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75d4a545789aa31cbcc811e86a921115ed397de9"},"cell_type":"code","source":"from keras import backend as K\nfrom keras.engine.topology import Layer, InputSpec\nfrom keras import initializers\n\nclass AttLayer(Layer):\n    def __init__(self, attention_dim):\n        self.init = initializers.get('normal')\n        self.supports_masking = True\n        self.attention_dim = attention_dim\n        super(AttLayer, self).__init__()\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n        self.W = K.variable(self.init((input_shape[-1], self.attention_dim)))\n        self.b = K.variable(self.init((self.attention_dim, )))\n        self.u = K.variable(self.init((self.attention_dim, 1)))\n        self.trainable_weights = [self.W, self.b, self.u]\n        super(AttLayer, self).build(input_shape)\n\n    def compute_mask(self, inputs, mask=None):\n        return mask\n\n    def call(self, x, mask=None):\n        # size of x :[batch_size, sel_len, attention_dim]\n        # size of u :[batch_size, attention_dim]\n        # uit = tanh(xW+b)\n        uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))\n        ait = K.dot(uit, self.u)\n        ait = K.squeeze(ait, -1)\n\n        ait = K.exp(ait)\n\n        if mask is not None:\n            # Cast the mask to floatX to avoid float64 upcasting in theano\n            ait *= K.cast(mask, K.floatx())\n        ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        ait = K.expand_dims(ait)\n        weighted_input = x * ait\n        output = K.sum(weighted_input, axis=1)\n\n        return output\n\n    def compute_output_shape(self, input_shape):\n        return (input_shape[0], input_shape[-1])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5d20f6b821431c92a9a3a551717ccf979304b508"},"cell_type":"markdown","source":"# MODELS\n1. CNN\n2. LSTM/GRU\n\nFor me LSTM work better than GRU"},{"metadata":{"trusted":true,"_uuid":"28fca0ae50902ed8657341594957ec4082bc0091"},"cell_type":"code","source":"def model_cnn(embedding_matrix):\n    filter_sizes = [1, 2, 3, 5]\n    num_filters = 64\n    inp = Input(shape=(maxlen, ))\n    embed = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.1)(embed)\n    \n    mpool = []\n    x = Reshape((maxlen, embed_size, 1))(x)\n    for fil in filter_sizes:\n        conv = Conv2D(num_filters, (fil, embed_size), kernel_initializer='he_normal', activation='relu')(x)\n        pool = MaxPool2D(pool_size=(maxlen - fil + 1, 1))(conv)\n        mpool.append(pool)\n        \n    x = Concatenate(axis=1)(mpool)\n    x = Flatten()(x)\n    x = Dropout(0.1)(x)\n    x = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inp, x)\n    return model\n    \n\ndef model_bgru(embedding_matrix):\n    inp = Input(shape=(maxlen, ))\n    embed = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = embed\n    x = SpatialDropout1D(0.2)(x)\n    \n    x1 = Bidirectional(CuDNNLSTM(64, return_sequences=True ))(x)\n    x2 = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x1)\n    x = Add()([x1, x2])\n    x = AttLayer(maxlen)(x)\n    x = Dropout(0.4)(x)\n    x = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inp, x)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ebaa818c963caecfe288d6aabc35b3a19a77a276"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_tra, X_val, y_tra, y_val = train_test_split(x_train, y, test_size = 0.1, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acb6b076bd64a6281946a96c91b339b2281daa60"},"cell_type":"code","source":"VAL_Y = []\nTEST_Y = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c99da33ff53779e0ebc508d32f9cc146df1c3cd"},"cell_type":"markdown","source":"# GLOVE EMBEDDING"},{"metadata":{"trusted":true,"_uuid":"dfde6fc25fa7f25eccbff0f496c7270259feb601"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8aa2dba63894771828bad7c960b4fad44da53a6b"},"cell_type":"markdown","source":"# GLOVE  - CNN"},{"metadata":{"trusted":true,"_uuid":"97a7f2dc62a0e14b1a2696757854c6ce2bd88d06","scrolled":true},"cell_type":"code","source":"model = model_cnn(embedding_matrix)\n#model.summary()\nmodel.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])\nbatch_size = 2048\nepochs = 5\nmodel.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)\nmodel.save('./model_cnn_glove.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"54d6ec99396a719e51d6c507b8c6236608c142b5"},"cell_type":"code","source":"val_pred_cnn_glove = model.predict([X_val], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_val, (val_pred_cnn_glove > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\n\ny_pred_cnn_glove = model.predict(x_test, batch_size=1024, verbose=True)\n\nVAL_Y.append(val_pred_cnn_glove)\nTEST_Y.append(y_pred_cnn_glove)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8acde13b562d6b651a674b61e71229ebacf97962"},"cell_type":"markdown","source":"# GLOVE  - BGRU"},{"metadata":{"trusted":true,"_uuid":"dd81282a79d1de0c89c64df093e97343ce6af550","scrolled":false},"cell_type":"code","source":"model = model_bgru(embedding_matrix)\n#model.summary()\nmodel.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])\nbatch_size = 2048\nepochs = 5\nmodel.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)\nmodel.save('./model_bgru_glove.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"5b42af7bcca059302d35f7bcbe69d514b5fc3410"},"cell_type":"code","source":"val_pred_bgru_glove = model.predict([X_val], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_val, (val_pred_bgru_glove > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\n\ny_pred_bgru_glove = model.predict(x_test, batch_size=1024, verbose=True)\n\nVAL_Y.append(val_pred_bgru_glove)\nTEST_Y.append(y_pred_bgru_glove)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"15a7e58c231fb06d736c3f823608c9e7488e500a"},"cell_type":"code","source":"del embeddings_index;\ndel embedding_matrix\ngc.collect()  ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"18e938f695b3cfb4ab30bf44cf7cc907a441dfd3"},"cell_type":"markdown","source":"# WIKI_NEWS - BGRU"},{"metadata":{"trusted":true,"_uuid":"fafe528936caada55c8722d3001507db144d1958"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"036431053e37297bccd2e11b8cd285441ed9bac3"},"cell_type":"code","source":"model = model_bgru(embedding_matrix)\n#model.summary()\nmodel.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])\nbatch_size = 2048\nepochs = 5\nmodel.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)\nmodel.save('./model_bgru_wiki.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"8e542536a5526bfa92ecfefefec30f27bffb4163"},"cell_type":"code","source":"val_pred_bgru_wiki = model.predict([X_val], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_val, (val_pred_bgru_wiki > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\n\ny_pred_bgru_wiki = model.predict(x_test, batch_size=1024, verbose=True)\n\nVAL_Y.append(val_pred_bgru_wiki)\nTEST_Y.append(y_pred_bgru_wiki)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bcecde29e1984dcd122003a66d4c7236803cd691"},"cell_type":"code","source":"del embeddings_index;\ndel embedding_matrix\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a81bf0b3a9cffd8e590c5fd8cdeee7a0f593db62"},"cell_type":"markdown","source":"# PARAGRAM - BGRU "},{"metadata":{"trusted":true,"_uuid":"bc87063d03fc100ab4de926aab3efe6d7e85de79"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b903998be692edde32432defe7736c388091cc8","scrolled":true},"cell_type":"code","source":"model = model_bgru(embedding_matrix)\n#model.summary()\nmodel.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3), metrics=['accuracy'])\nbatch_size = 2048\nepochs = 5\nmodel.fit(X_tra, y_tra, batch_size=batch_size, epochs=epochs, validation_data=(X_val, y_val), verbose=True)\nmodel.save('./model_bgru_paragram.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"506e977f5a88e401c169f42d9cf206aa6594426e","scrolled":true},"cell_type":"code","source":"val_pred_bgru_paragram = model.predict([X_val], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_val, (val_pred_bgru_paragram > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)\n\ny_pred_bgru_paragram = model.predict(x_test, batch_size=1024, verbose=True)\n\nVAL_Y.append(val_pred_bgru_paragram)\nTEST_Y.append(y_pred_bgru_paragram)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"76684efd4a89371cb6e5bfd1c9e5a6633c7b9283"},"cell_type":"markdown","source":"# Concat Result & Best Threshold"},{"metadata":{"trusted":true,"_uuid":"9d7e61ed6bd04ad3d8b79fda77a450f73d2f4ae9","scrolled":true},"cell_type":"code","source":"\npred_val_y = (2.5*val_pred_cnn_glove + 2.5*val_pred_bgru_glove + 2.5*val_pred_bgru_wiki + 2.5*val_pred_bgru_paragram)/10\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(y_val, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f2fc9255ea014db50828d829a026536aa0e07f2"},"cell_type":"markdown","source":"# Submission"},{"metadata":{"trusted":true,"_uuid":"b323d365f21b1e82d59dda99e9acfb3e1c8366b1"},"cell_type":"code","source":"# y_pred = 0\n# for i in range(len(y_th)):\n#     y_pred += (y_th[i] * TEST_Y[i])\n# y_pred = y_pred / 10\n\ny_pred = (2.5*y_pred_cnn_glove + 2.5*y_pred_bgru_glove + 2.5*y_pred_bgru_wiki + 2.5*y_pred_bgru_paragram)/10\n\ny_te = (y_pred[:,0] > best_thresh).astype(np.int)\n\nsubmit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\nsubmit_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6de2f013596f995295808b74fd93027f19acbc8"},"cell_type":"code","source":"from IPython.display import HTML\nimport base64  \nimport pandas as pd  \n\ndef create_download_link( df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv(index =False)\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\ncreate_download_link(submit_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a45f1abebefefa640dad715ca3ea54b4a61705f5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea432a03f46721989581482bcbdc0f3e89139e31"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47ffbe0420b79872fe6a86a2387cae63c239bcdc"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}