{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\n\nSEED = 2018\n\nnp.random.seed(SEED)\ntf.set_random_seed(SEED)\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import *\nfrom tensorflow.keras.models import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n\nfrom tensorflow.keras import backend as K\n#from keras.engine.topology import Layer\nfrom tensorflow.keras.layers import Layer\nfrom tensorflow.keras import initializers, regularizers, constraints, optimizers, layers\n\nimport gc, re\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"876c340c9d3b711d89481748e2772821821a6a22"},"cell_type":"code","source":"# https://www.kaggle.com/theoviel/improve-your-score-with-some-text-preprocessing/notebook\ncontraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\nmispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\npunct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }    \n\ndef clean_text(x):\n    for dic in [contraction_mapping, mispell_dict, punct_mapping]:\n        for word in dic.keys():\n            x = x.replace(word, dic[word])\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93abdca775bbf841f5618870199e3924fbcbb5e7"},"cell_type":"code","source":"def load_and_preprocess_data(max_features=50000, maxlen=70):\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n    ## split to train and val\n    train_df, val_df = train_test_split(train_df, test_size=0.1, random_state=SEED)\n\n    train_df['question_text'] = train_df['question_text'].fillna(\"\").apply(lambda x: clean_text(x))\n    test_df['question_text'] = test_df['question_text'].fillna(\"\").apply(lambda x: clean_text(x))\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    val_X = val_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n    \n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X) + list(test_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    val_X = tokenizer.texts_to_sequences(val_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n    \n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    val_X = pad_sequences(val_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n    \n    ## Get the target values\n    train_y = train_df['target'].values\n    val_y = val_df['target'].values  \n    \n    #shuffling the data\n    np.random.seed(2018)\n    trn_idx = np.random.permutation(len(train_X))\n    val_idx = np.random.permutation(len(val_X))\n\n    train_X = train_X[trn_idx]\n    val_X = val_X[val_idx]\n    train_y = train_y[trn_idx]\n    val_y = val_y[val_idx]    \n    \n    return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"161a52be4aa37b1b7bfdbe8dec2297d679b52cb5"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63b8f1f3bff12df4d931ea46e26040a0464bd2c4"},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight('{}_W'.format(self.name), shape=(input_shape[-1].value,),\n                                 initializer=self.init,\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight('{}_b'.format(self.name), shape=(input_shape[1].value,),\n                                     initializer='zero',\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint\n                                    )\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a0653988ce0d12e356f19b78b2a7ca2c6414d09d"},"cell_type":"code","source":"# def cnn1d(embedding_matrix, maxlen=70, max_features=50000, units=512):\n#     inp = Input(shape=(maxlen,))\n#     x = Embedding(max_features, embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False)(inp)\n    \n#     layer_conv3 = tf.keras.layers.Conv1D(units, 3, activation=\"relu\")(x)\n#     layer_conv3 = tf.keras.layers.GlobalMaxPooling1D()(layer_conv3)\n\n#     layer_conv4 = tf.keras.layers.Conv1D(units, 2, activation=\"relu\")(x)\n#     layer_conv4 = tf.keras.layers.GlobalMaxPooling1D()(layer_conv4)\n\n#     layer = tf.keras.layers.concatenate([layer_conv4, layer_conv3], axis=1)\n#     layer = tf.keras.layers.BatchNormalization()(layer)\n#     layer = tf.keras.layers.Dropout(0.1)(layer)\n\n#     output = tf.keras.layers.Dense(1, activation=\"sigmoid\")(layer)\n    \n#     model = Model(inputs=inp, outputs=output)\n#     model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])\n#     return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78132bfbc5e06e768252919880f34dd51a9b1418"},"cell_type":"code","source":"def model_lstm_atten(embedding_matrix, maxlen=70, max_features=50000, units=64):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNLSTM(units*2, return_sequences=True))(x)\n    x = Bidirectional(CuDNNLSTM(units, return_sequences=True))(x)\n    x = Attention(maxlen)(x)\n    x = Dense(units, activation=\"relu\")(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad0c6ccf1351acb4f580866f2c6873ee939a8122"},"cell_type":"code","source":"def model_lstm_du(embedding_matrix, maxlen=70, max_features=50000, units=64):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embedding_matrix.shape[1], weights=[embedding_matrix])(inp)\n    x = Bidirectional(CuDNNGRU(units, return_sequences=True))(x)\n    attn = Attention(maxlen)(x)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    conc = concatenate([avg_pool, max_pool, attn])\n    conc = Dense(units, activation=\"relu\")(conc)\n    conc = Dropout(0.1)(conc)\n    outp = Dense(1, activation=\"sigmoid\")(conc)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77c4f25a2fe481f37dd2dece7782e7ad038eed2d"},"cell_type":"code","source":"def model_gru_atten_3(embedding_matrix, maxlen=70, max_features=50000, units=64):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNGRU(units*2, return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(units, return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(units//2, return_sequences=True))(x)\n    x = Attention(maxlen)(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e2975b4623ce850f033a4feeb6f939e9461edf1"},"cell_type":"code","source":"def model_bilstm_2dcnn(embedding_matrix, maxlen=70, max_features=50000, units=64):\n    conv_filters = 32\n    inp = Input(shape=(maxlen,), dtype='int32')\n    x = Embedding(max_features, embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.1)(x)\n    x = Bidirectional(CuDNNLSTM(units, return_sequences=True))(x)\n    x = Dropout(0.1)(x)\n    x = Reshape((2 * maxlen, units, 1))(x)\n    x = Conv2D(conv_filters, (3, 3))(x)\n    x = MaxPool2D(pool_size=(2, 2))(x)\n    x = Flatten()(x)\n    outp= Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7dfc811f8587c8522dfaa1a8570345a89801f1fe"},"cell_type":"code","source":"def train_pred(model, train_X, train_y, val_X, val_y, epochs=2):\n    for e in range(epochs):\n        model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))\n        pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)\n\n        best_thresh = 0.5\n        best_score = 0.0\n        for thresh in np.arange(0.1, 0.501, 0.01):\n            thresh = np.round(thresh, 2)\n            score = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n            if score > best_score:\n                best_thresh = thresh\n                best_score = score\n\n        print(\"Val F1 Score: {:.4f}\".format(best_score))\n\n    pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)\n    print('='*100)\n    return pred_val_y, pred_test_y, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7bca4c6dc6e2b21e73efb5082c33b888030ec59"},"cell_type":"code","source":"embed_size = 300\nmax_features = 95000\nmaxlen = 40","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7de762fcd44b0192b8d8d93cbeeb88ddb990fcb5"},"cell_type":"code","source":"import time\nstart_time = time.time()\n\ntrain_X, val_X, test_X, train_y, val_y, word_index = load_and_preprocess_data(max_features=max_features, maxlen=maxlen)\n\ntotal_time = (time.time() - start_time)/60.0\nprint(\"Took {0} minutes\".format(total_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"de1d72962abc67661e56fb80a35334c146fd49b4"},"cell_type":"code","source":"# all_len = []\n\n# for x in train_X:\n#     l = 0\n#     for dx in x:\n#         if dx > 0:\n#             l += 1\n#     all_len.append(l)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"381357ba38672001d175581a1aa71605c0b1e42c"},"cell_type":"code","source":"# max_tokens = np.mean(all_len)+2 * np.std(all_len)\n# print(max_tokens)\n# p = np.sum(all_len < max_tokens)/len(all_len)\n# print(p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45d96fe50e5f76387e117440a7f3a12f463822f9"},"cell_type":"code","source":"import time\nstart_time = time.time()\n\nembedding_matrix_1 = load_glove(word_index)\nembedding_matrix_2 = load_fasttext(word_index)\nembedding_matrix_3 = load_para(word_index)\n\ntotal_time = (time.time() - start_time)/60.0\nprint(\"Took {0} minutes\".format(total_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef7f64055336589446a4ce664ff62ad4a2df641b"},"cell_type":"code","source":"#embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)\nembedding_matrix = np.concatenate((embedding_matrix_1, embedding_matrix_2, embedding_matrix_3), axis=1)\nprint(np.shape(embedding_matrix))\n\ndel embedding_matrix_2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60e4e730c671e2258582dfda09612e88a96942f9"},"cell_type":"code","source":"outputs = []","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"535fee9302d9f8cd35dab173c88254653845f867","scrolled":true},"cell_type":"code","source":"# model = cnn1d(embedding_matrix, maxlen=maxlen, max_features=max_features, units=128)\n# pred_val_y, pred_test_y, best_score = train_pred(model, train_X, train_y, val_X, val_y, epochs = 2)\n# outputs.append([pred_val_y, pred_test_y, best_score, 'CNN 1D'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2e692769ea57dcb1256ba15621381c55aa0a80b5"},"cell_type":"code","source":"model = model_lstm_atten(embedding_matrix, maxlen=maxlen, max_features=max_features, units=64)\npred_val_y, pred_test_y, best_score = train_pred(model, train_X, train_y, val_X, val_y, epochs = 3)\noutputs.append([pred_val_y, pred_test_y, best_score, 'LSTM ATTN'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3560bfc25d8fad9cafee1f03d18a2ffd43acf459"},"cell_type":"code","source":"model = model_lstm_atten(embedding_matrix_3, maxlen=maxlen, max_features=max_features, units=64)\npred_val_y, pred_test_y, best_score = train_pred(model, train_X, train_y, val_X, val_y, epochs = 2)\noutputs.append([pred_val_y, pred_test_y, best_score, 'LSTM ATTN PARA'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ac0b77c1d58b54ec2c03faca1a1f29ed3f508b9"},"cell_type":"code","source":"model = model_lstm_du(embedding_matrix, maxlen=maxlen, max_features=max_features, units=64)\npred_val_y, pred_test_y, best_score = train_pred(model, train_X, train_y, val_X, val_y, epochs = 1)\noutputs.append([pred_val_y, pred_test_y, best_score, 'LSTM DU'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"350513145184966ec320e63c24b713276557c499"},"cell_type":"code","source":"model = model_lstm_du(embedding_matrix_1, maxlen=maxlen, max_features=max_features, units=64)\npred_val_y, pred_test_y, best_score = train_pred(model, train_X, train_y, val_X, val_y, epochs = 1)\noutputs.append([pred_val_y, pred_test_y, best_score, 'LSTM DU GLOVE'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a02ddbf9018eab230da0c355667366be29cd0c92"},"cell_type":"code","source":"model = model_lstm_du(embedding_matrix_3, maxlen=maxlen, max_features=max_features, units=64)\npred_val_y, pred_test_y, best_score = train_pred(model, train_X, train_y, val_X, val_y, epochs = 2)\noutputs.append([pred_val_y, pred_test_y, best_score, 'LSTM DU PARA'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c34dfbce19e54a18f62d598653e05bb6e72739c8"},"cell_type":"code","source":"model = model_gru_atten_3(embedding_matrix, maxlen=maxlen, max_features=max_features, units=64)\npred_val_y, pred_test_y, best_score = train_pred(model, train_X, train_y, val_X, val_y, epochs = 2)\noutputs.append([pred_val_y, pred_test_y, best_score, 'GRU ATTN'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13568f3cfe894a24610b21ac64320ab062365285"},"cell_type":"code","source":"model = model_gru_atten_3(embedding_matrix_1, maxlen=maxlen, max_features=max_features, units=64)\npred_val_y, pred_test_y, best_score = train_pred(model, train_X, train_y, val_X, val_y, epochs = 3)\noutputs.append([pred_val_y, pred_test_y, best_score, 'GRU ATTN GLOVE'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f34c7ce2d37e49edc79680692cc5c8507a40b545"},"cell_type":"code","source":"model = model_bilstm_2dcnn(embedding_matrix, maxlen=maxlen, max_features=max_features, units=64)\npred_val_y, pred_test_y, best_score = train_pred(model, train_X, train_y, val_X, val_y, epochs = 3)\noutputs.append([pred_val_y, pred_test_y, best_score, 'BILSTM CNN2D'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2058dd6bba02bfbe934b7ec0b1f1d586e853cfe"},"cell_type":"code","source":"weights = [i for i in range(1, len(outputs) + 1)]\nweights = [float(i) / sum(weights) for i in weights] \nprint(weights)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ce44bb9e2bb8bbecd31c5a1a7cccb068aebda3d"},"cell_type":"code","source":"for output in outputs:\n    print(output[2], output[3])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"058a1987ddc2e0fb2549b623efc2121f5a1749dd"},"cell_type":"code","source":"from sklearn.linear_model import LinearRegression\nX = np.asarray([outputs[i][0] for i in range(len(outputs))])\nX = X[...,0]\nreg = LinearRegression().fit(X.T, val_y)\nprint(reg.score(X.T, val_y),reg.coef_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa40f150c96bcd6346603fb22c0f39eb00b741c7","scrolled":true},"cell_type":"code","source":"#Weights\npred_val_y = np.sum([outputs[i][0] * weights[i] for i in range(0, len(outputs))], axis = 0)\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nprint(\"Best threshold: \", thresholds[0][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a15d9c379b8b51dc3d484bdb885fc86247dce15","scrolled":true},"cell_type":"code","source":"# Mean\npred_val_y = np.mean([outputs[i][0] for i in range(len(outputs))], axis = 0)\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nthresholds.sort(key=lambda x: x[1], reverse=True)\nprint(\"Best threshold: \", thresholds[0][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3445779418e3f18c8ccc751ab1937ad871ca57f","scrolled":true},"cell_type":"code","source":"# Regression Coeffecient\npred_val_y = np.sum([outputs[i][0] * reg.coef_[i] for i in range(0, len(outputs))], axis = 0)\n\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db802502d3da771dfb278550834b21a97a694595"},"cell_type":"code","source":"pred_test_y = np.sum([outputs[i][1] * reg.coef_[i] for i in range(0, len(outputs))], axis = 0)\n#pred_test_y = np.mean([outputs[i][1] for i in range(len(outputs))], axis = 0)\n\npred_test_y = (pred_test_y > best_thresh).astype(int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca0c7e8472b52b34e27c91dd49bccfd5e213d6e7"},"cell_type":"code","source":"from IPython.display import HTML\nimport base64  \nimport pandas as pd  \n\ndef create_download_link( df, title = \"Download CSV file\", filename = \"data.csv\"):  \n    csv = df.to_csv(index =False)\n    b64 = base64.b64encode(csv.encode())\n    payload = b64.decode()\n    html = '<a download=\"{filename}\" href=\"data:text/csv;base64,{payload}\" target=\"_blank\">{title}</a>'\n    html = html.format(payload=payload,title=title,filename=filename)\n    return HTML(html)\n\ncreate_download_link(out_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05ad8a59578eee2d1186b6fb467d1fb3f3fd51e9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}