{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport gc\nimport re\nimport operator \nimport string\nfrom tqdm import tqdm\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *\nfrom keras.models import Model\nfrom keras.engine.topology import Layer\nfrom gensim.models import KeyedVectors\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score, classification_report, accuracy_score\nfrom sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis, QuadraticDiscriminantAnalysis\nfrom sklearn.naive_bayes import GaussianNB\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.linear_model import LogisticRegression, LinearRegression\nfrom sklearn.ensemble import VotingClassifier, BaggingClassifier\nimport xgboost as xgb\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"embedding_size = 300 # how big is each word vector\nmax_features = 95000 # how many unique words to use (i.e num rows in embedding vector)\nmax_len = 100 # max number of words in a question to use","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7f239e877f397fe93911336e5de18d0bec14b1f3"},"cell_type":"code","source":"def clean_text(x):\n    x = str(x)\n    for p in string.punctuation + '“…':\n        x = x.replace(p, ' ' + p + ' ')\n    \n    x = x.replace('_', '')\n    \n    x = re.sub(\"`\",\"'\", x)\n    x = re.sub(\"(?i)n\\'t\",' not', x)\n    x = re.sub(\"(?i)\\'re\",' are', x)\n    x = re.sub(\"(?i)\\'s\",' is', x)\n    x = re.sub(\"(?i)\\'d\",' would', x)\n    x = re.sub(\"(?i)\\'ll\",' will', x)\n    x = re.sub(\"(?i)\\'t\",' not', x)\n    x = re.sub(\"(?i)\\'ve\",' have', x)\n    x = re.sub(\"(?i)\\'m\",' am', x)\n    \n    x = re.sub(\"(?i)n\\’t\",' not', x)\n    x = re.sub(\"(?i)\\’re\",' are', x)\n    x = re.sub(\"(?i)\\’s\",' is', x)\n    x = re.sub(\"(?i)\\’d\",' would', x)\n    x = re.sub(\"(?i)\\’ll\",' will', x)\n    x = re.sub(\"(?i)\\’t\",' not', x)\n    x = re.sub(\"(?i)\\’ve\",' have', x)\n    x = re.sub(\"(?i)\\’m\",' am', x)\n    \n    x = re.sub('(?i)Quorans','Quora', x)\n    x = re.sub('(?i)Qoura','Quora', x)\n    x = re.sub('(?i)Quoran','Quora', x)\n    x = re.sub('(?i)dropshipping','drop shipping', x)\n    x = re.sub('(?i)HackerRank','Hacker Rank', x)\n    x = re.sub('(?i)Unacademy','un academy', x)\n    x = re.sub('(?i)eLitmus','India hire employees', x)\n    x = re.sub('(?i)WooCommerce','Commerce', x)\n    x = re.sub('(?i)hairfall','hair fall', x)\n    x = re.sub('(?i)marksheet','mark sheet', x)\n    x = re.sub('(?i)articleship','article ship', x)\n    x = re.sub('(?i)cryptocurrencies','cryptocurrency', x)\n    x = re.sub('(?i)coinbase','cryptocurrency', x)\n    x = re.sub('(?i)altcoin','bitcoin', x)\n    x = re.sub('(?i)altcoins','bitcoins', x)\n    x = re.sub('(?i)litecoin','bitcoin', x)\n    x = re.sub('(?i)litecoins','bitcoins', x)\n    x = re.sub('(?i)demonetisation','demonetization', x)\n    x = re.sub('(?i)ethereum','bitcoin', x)\n    x = re.sub('(?i)ethereums','bitcoins', x)\n    x = re.sub('(?i)quorans','quora', x)\n    x = re.sub('(?i)Brexit','britan exit', x)\n    x = re.sub('(?i)upwork','freelance', x)\n    x = re.sub('(?i)Unacademy','un academy', x)\n    x = re.sub('(?i)Blockchain','blockchain', x)\n    x = re.sub('(?i)GDPR','General Data Protection Regulation', x)\n    x = re.sub('(?i)Qoura','quora', x)\n    x = re.sub('(?i)HackerRank','Hacker Rank', x)\n    x = re.sub('(?i)Cryptocurrency','cryptocurrency', x)\n    x = re.sub('(?i)Binance','cryptocurrency', x)\n    x = re.sub('(?i)Redmi','mobile phone', x)\n    x = re.sub('(?i)TensorFlow','Tensor Flow', x)\n    x = re.sub('(?i)Golang','programming language', x)\n    x = re.sub('(?i)eLitmus','India hire employees', x)\n    \n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab360df6081e1ea796f9dafe9d072e69aec1d1e6"},"cell_type":"code","source":"def get_coefs(word,*arr): \n    return word, np.asarray(arr, dtype='float32')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43492bfc9641050b83870b53c6352657cf921d89"},"cell_type":"code","source":"# https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"46182b3e08a5bef618755a76208f1305f10eb710"},"cell_type":"code","source":"def get_model_1(embedding_matrix):\n    inp = Input(shape=(max_len, ))\n    x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNGRU(80, return_sequences=True))(x)\n    x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)\n    \n    y = Conv1D(64, kernel_size=2, padding='valid', kernel_initializer='he_uniform')(x)\n    y = Dense(16, activation='relu')(y)\n    y = Flatten()(y)\n    \n    atten = Attention(max_len)(x)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    \n    conc = concatenate([atten, avg_pool, max_pool])\n#     conc = Conv2D(64, kernel_size=2, padding='valid', kernel_initializer='he_uniform')(conc)\n    conc = Dense(64, activation='relu')(conc)\n    \n    conc_z = concatenate([conc, y])\n    conc_z = Dense(32, activation='relu')(conc_z)\n    conc_z = Dropout(0.1)(conc_z)\n    output = Dense(1, activation='sigmoid')(conc_z)\n    \n    model = Model(inputs=inp, outputs=output)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a4254c233e4bde0de524e4308c66096254e0102"},"cell_type":"code","source":"# https://www.kaggle.com/shujian/fork-of-mix-of-nn-models\n\ndef get_model_2(embedding_matrix):\n    inp = Input(shape=(max_len,))\n    x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(100, return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n    x = Attention(max_len)(x)\n    output = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=inp, outputs=output)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b0f1e8a091fe9cbad6414bd30a39129ca1f14ab"},"cell_type":"code","source":"# https://www.kaggle.com/yekenot/2dcnn-textclassifier\n    \ndef get_model_3(embedding_matrix):\n    filter_sizes = [1,2,3,5]\n    num_filters = 42\n\n    inp = Input(shape=(max_len, ))\n    x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Reshape((max_len, embedding_size, 1))(x)\n    \n    conv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embedding_size),\n                                 kernel_initializer='he_normal', activation='tanh')(x)\n    conv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embedding_size),\n                                 kernel_initializer='he_normal', activation='tanh')(x)\n    conv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embedding_size), \n                                 kernel_initializer='he_normal', activation='tanh')(x)\n    conv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embedding_size),\n                                 kernel_initializer='he_normal', activation='tanh')(x)\n    \n    maxpool_0 = MaxPool2D(pool_size=(max_len - filter_sizes[0] + 1, 1))(conv_0)\n    maxpool_1 = MaxPool2D(pool_size=(max_len - filter_sizes[1] + 1, 1))(conv_1)\n    maxpool_2 = MaxPool2D(pool_size=(max_len - filter_sizes[2] + 1, 1))(conv_2)\n    maxpool_3 = MaxPool2D(pool_size=(max_len - filter_sizes[3] + 1, 1))(conv_3)\n        \n    z = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])   \n    z = Flatten()(z)\n    z = Dropout(0.1)(z) \n    output = Dense(1, activation='sigmoid')(z)\n    \n    model = Model(inputs=inp, outputs=output)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ae8f0b94c1277865936cc61a7c4755be2464610"},"cell_type":"code","source":"# https://www.kaggle.com/shujian/single-rnn-model-with-meta-features?scriptVersionId=7593124\n\ndef get_model_4(embedding_matrix):\n    inp = Input(shape=(max_len, ))\n    x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNLSTM(40, return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(40, return_sequences=True))(x)\n    \n    atten = Attention(max_len)(x)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    \n    conc = concatenate([atten, avg_pool, max_pool])\n    conc = Dense(64, activation='relu')(conc)\n    conc = Dropout(0.1)(conc)\n    output = Dense(1, activation='sigmoid')(conc)\n    \n    model = Model(inputs=inp, outputs=output)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a74e53ed7637d5ce1d9bdd37e3bafd03704c0315"},"cell_type":"code","source":"# https://www.kaggle.com/suicaokhoailang/beating-the-baseline-with-one-weird-trick-0-691\n\ndef get_model_5(embedding_matrix):\n    inp = Input(shape=(max_len,))\n    x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n    x = Attention(max_len)(x)\n    x = Dense(16, activation='relu')(x)\n    x = Dropout(0.1)(x)\n    output = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=inp, outputs=output)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f1edd20178d3cdedfa59bb06c9b7f1bc37fa8c8"},"cell_type":"code","source":"def get_model_6(embedding_matrix):\n    inp = Input(shape=(max_len, ))\n    x = Embedding(max_features, embedding_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.15)(x)\n    x = Bidirectional(CuDNNLSTM(max_len, return_sequences=True))(x)\n    x = Conv1D(64, kernel_size=2, padding='valid', kernel_initializer='he_uniform')(x)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    conc = concatenate([avg_pool, max_pool])\n    output = Dense(1, activation='sigmoid')(conc)\n    \n    model = Model(inputs=inp, outputs=output)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9bd348640dcf1812655b7c11287c9cb61debff2"},"cell_type":"code","source":"def get_f1_metric(y_true, y_predicted):        \n    f1_metrics = []\n    for thresh in np.arange(0.1, 0.51, 0.01):\n        thresh = np.round(thresh, 2)\n        f1_metric = f1_score(y_true, (y_predicted>thresh).astype(int))\n        f1_metrics.append((thresh, f1_metric))\n        print('F1 score at threshold {0} is {1}'.format(thresh, f1_metric))\n    \n    threshold = max(f1_metrics, key=lambda x: x[1])[0]\n    \n    print(threshold)\n    \n    return threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9af3b0acf8cdaf2a54b698432cc41e5a46100a09"},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')\ntest_df = pd.read_csv('../input/test.csv')\n\nprint('Train shape : ', train_df.shape)\nprint('Test shape : ', test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"47663e068a62d28bf082fe99c3f57412fcd88ef9"},"cell_type":"code","source":"train_df['question_text'] = train_df['question_text'].progress_apply(lambda x: clean_text(x))\ntest_df['question_text'] = test_df['question_text'].progress_apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b2b83c03cfe0eeea013ee3e978fefe6f5268c65"},"cell_type":"code","source":"train_df, val_df = train_test_split(train_df, \n                                    test_size=0.07, \n                                    random_state=42, \n                                    stratify=train_df['target'])\n\nX_train = train_df['question_text'].values\ny_train = train_df['target'].values\nX_val = val_df['question_text'].values\ny_val = val_df['target'].values\nX_test = test_df['question_text'].values\n\nprint(X_train.shape, \n      y_train.shape, \n      X_val.shape, \n      y_val.shape, \n      X_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3cb006964a6f3c804565cd4de72f9202471d0dd0"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train) + list(X_val))\nX_train = tokenizer.texts_to_sequences(X_train)\nX_val = tokenizer.texts_to_sequences(X_val)\nX_test = tokenizer.texts_to_sequences(X_test)\n\nX_train = pad_sequences(X_train, maxlen=max_len)\nX_val = pad_sequences(X_val, maxlen=max_len)\nX_test = pad_sequences(X_test, maxlen=max_len)\n\nword_index = tokenizer.word_index\n\nprint(X_train.shape, X_val.shape, X_test.shape, len(word_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4cf189b1fad88bb60eb0632556cf09d9e915d060"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nembeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nnb_words = min(max_features, len(word_index))\nembedding_matrix_glove = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in tqdm(word_index.items()):\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_glove[i] = embedding_vector\n\ndel embeddings_index\ngc.collect() \n\nprint(embedding_matrix_glove.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e29fd63a75b60d56b7a0b388563a6fa5df1dede"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\nembeddings_index = dict(get_coefs(*o.split(' ')) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nnb_words = min(max_features, len(word_index))\nembedding_matrix_wiki = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in tqdm(word_index.items()):\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_wiki[i] = embedding_vector\n\ndel embeddings_index, word_index\ngc.collect()   \n\nprint(embedding_matrix_wiki.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33b259a07601bfef30537a81b299df626e8f3a32"},"cell_type":"code","source":"embedding_matrix = np.mean([embedding_matrix_glove, embedding_matrix_wiki], axis = 0)\nnp.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6c0092ebaab0e2e31e50dd010284573e03d6861e"},"cell_type":"code","source":"model_1 = get_model_1(embedding_matrix_glove)\nmodel_1.fit(X_train, y_train, batch_size=512, epochs=3, validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fce44be2c8abb28ac5d6628a3525df47bb53ac06"},"cell_type":"code","source":"model_2 = get_model_2(embedding_matrix_glove)\nmodel_2.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1df2ea3391c592b840c2873cc39262bea9fa1765"},"cell_type":"code","source":"model_3 = get_model_3(embedding_matrix)\nmodel_3.fit(X_train, y_train, batch_size=512, epochs=3, validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92735d9442216a7034e07a6a6ed051d7dae8794b"},"cell_type":"code","source":"model_4 = get_model_4(embedding_matrix)\nmodel_4.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77ab9da1308a370521dba4391fc0fd393a8f6d04"},"cell_type":"code","source":"model_5 = get_model_5(embedding_matrix_wiki)\nmodel_5.fit(X_train, y_train, batch_size=512, epochs=3, validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bcbb4613a7887a4ca80e14f855265192edd0415"},"cell_type":"code","source":"model_6 = get_model_6(embedding_matrix_wiki)\nmodel_6.fit(X_train, y_train, batch_size=512, epochs=2, validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58f687941df2e84b14305010c719051671a37a66"},"cell_type":"code","source":"pred_val_1 = model_1.predict([X_val], batch_size=512, verbose=1)\npred_val_2 = model_2.predict([X_val], batch_size=512, verbose=1)\npred_val_3 = model_3.predict([X_val], batch_size=512, verbose=1)\npred_val_4 = model_4.predict([X_val], batch_size=512, verbose=1)\npred_val_5 = model_5.predict([X_val], batch_size=512, verbose=1)\npred_val_6 = model_6.predict([X_val], batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d1369b42cf7bb77d6fb02e0b53a1e5d81648f13"},"cell_type":"code","source":"pred_val_1_df = np.reshape(pred_val_1, (pred_val_1.shape[0]))\npred_val_2_df = np.reshape(pred_val_2, (pred_val_2.shape[0]))\npred_val_3_df = np.reshape(pred_val_3, (pred_val_3.shape[0]))\npred_val_4_df = np.reshape(pred_val_4, (pred_val_4.shape[0]))\npred_val_5_df = np.reshape(pred_val_5, (pred_val_5.shape[0]))\npred_val_6_df = np.reshape(pred_val_6, (pred_val_6.shape[0]))\npred_val_df = np.reshape(y_val, (y_val.shape[0]))\n\nvalidation_df = pd.DataFrame({'val_1': pred_val_1_df, 'val_2': pred_val_2_df, 'val_3': pred_val_3_df, 'val_4': pred_val_4_df, 'val_5': pred_val_5_df, 'val_6': pred_val_6_df, 'prediction': pred_val_df})\nvalidation_df.to_csv('validation.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a6e52402559505da901c79b7781460e5f9a823e"},"cell_type":"code","source":"validation_df['val_12_mean'] = (validation_df['val_1'] + validation_df['val_2']) / 2.0\nvalidation_df['val_34_mean'] = (validation_df['val_3'] + validation_df['val_4']) / 2.0\nvalidation_df['val_56_mean'] = (validation_df['val_5'] + validation_df['val_6']) / 2.0\nvalidation_df['val_123_mean'] = (validation_df['val_1'] + validation_df['val_2'] + validation_df['val_3']) / 3.0\nvalidation_df['val_456_mean'] = (validation_df['val_4'] + validation_df['val_5'] + validation_df['val_6']) / 3.0\nvalidation_df['val_1_log'] = np.log(validation_df['val_1'])\nvalidation_df['val_2_log'] = np.log(validation_df['val_2'])\nvalidation_df['val_3_log'] = np.log(validation_df['val_3'])\nvalidation_df['val_4_log'] = np.log(validation_df['val_4'])\nvalidation_df['val_5_log'] = np.log(validation_df['val_5'])\nvalidation_df['val_6_log'] = np.log(validation_df['val_6'])\nvalidation_df['val_12_log_mean'] = (validation_df['val_1_log'] + validation_df['val_2_log']) / 2.0\nvalidation_df['val_34_log_mean'] = (validation_df['val_3_log'] + validation_df['val_4_log']) / 2.0\nvalidation_df['val_56_log_mean'] = (validation_df['val_5_log'] + validation_df['val_6_log']) / 2.0\nvalidation_df['val_123_log_mean'] = (validation_df['val_1_log'] + validation_df['val_2_log'] + validation_df['val_3_log']) / 3.0\nvalidation_df['val_456_log_mean'] = (validation_df['val_4_log'] + validation_df['val_5_log'] + validation_df['val_6_log']) / 3.0\n\nvalidation_df = validation_df[[c for c in validation_df.columns if c != 'prediction'] + ['prediction']]\n\nvalidation_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"52538142c617d9c1b5b2aef45fe149133dd0acd6"},"cell_type":"code","source":"X_val = validation_df[validation_df.columns.values[:-1]].values\ny_val = validation_df[validation_df.columns.values[-1]].values\n\nclf = VotingClassifier(estimators=\n                       [('gnb', GaussianNB()), \n                        ('xgb', xgb.XGBClassifier(max_dept=100, n_estimators=15, learning_rate=0.05, \n                                                  colsample_bytree=0.5, gamma=0.01, reg_alpha=4, \n                                                  objective='binary:logistic')), \n                        ('knn', KNeighborsClassifier(n_neighbors=500)), \n                        ('rf', RandomForestClassifier(max_depth=3, n_estimators=100)), \n                        ('lr', LogisticRegression()), \n                        ('dt', DecisionTreeClassifier(max_depth=3, criterion='entropy')),\n                        ('adb', AdaBoostClassifier(base_estimator=DecisionTreeClassifier(max_depth=3, criterion='entropy'), learning_rate=0.05)),\n                        ('gb', GradientBoostingClassifier(max_depth=3, learning_rate=0.05, \n                                                          n_estimators=100)),\n                        ('qda', QuadraticDiscriminantAnalysis()),\n                        ('lda', LinearDiscriminantAnalysis())\n                       ],\n                       voting='soft')\nclf.fit(X_val, y_val)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2336f7d39e484bc4f3ee2b9e4235fae84a6223e"},"cell_type":"code","source":"pred_test_1 = model_1.predict([X_test], batch_size=512, verbose=1)\npred_test_2 = model_2.predict([X_test], batch_size=512, verbose=1)\npred_test_3 = model_3.predict([X_test], batch_size=512, verbose=1)\npred_test_4 = model_4.predict([X_test], batch_size=512, verbose=1)\npred_test_5 = model_5.predict([X_test], batch_size=512, verbose=1)\npred_test_6 = model_6.predict([X_test], batch_size=512, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e2ab9359bc571090c0ae86ea9240eb3d7fd2cbdb"},"cell_type":"code","source":"pred_test_1_df = np.reshape(pred_test_1, (pred_test_1.shape[0]))\npred_test_2_df = np.reshape(pred_test_2, (pred_test_2.shape[0]))\npred_test_3_df = np.reshape(pred_test_3, (pred_test_3.shape[0]))\npred_test_4_df = np.reshape(pred_test_4, (pred_test_4.shape[0]))\npred_test_5_df = np.reshape(pred_test_5, (pred_test_5.shape[0]))\npred_test_6_df = np.reshape(pred_test_6, (pred_test_6.shape[0]))\n\ntest_df1 = pd.DataFrame({'val_1': pred_test_1_df, 'val_2': pred_test_2_df, 'val_3': pred_test_3_df, 'val_4': pred_test_4_df, 'val_5': pred_test_5_df, 'val_6': pred_test_6_df})\ntest_df1.to_csv('test.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ae127e862d439bbddcfea61b954f1df137b95c1"},"cell_type":"code","source":"test_df1['val_12_mean'] = (test_df1['val_1'] + test_df1['val_2']) / 2.0\ntest_df1['val_34_mean'] = (test_df1['val_3'] + test_df1['val_4']) / 2.0\ntest_df1['val_56_mean'] = (test_df1['val_5'] + test_df1['val_6']) / 2.0\ntest_df1['val_123_mean'] = (test_df1['val_1'] + test_df1['val_2'] + test_df1['val_3']) / 3.0\ntest_df1['val_456_mean'] = (test_df1['val_4'] + test_df1['val_5'] + test_df1['val_6']) / 3.0\ntest_df1['val_1_log'] = np.log(test_df1['val_1'])\ntest_df1['val_2_log'] = np.log(test_df1['val_2'])\ntest_df1['val_3_log'] = np.log(test_df1['val_3'])\ntest_df1['val_4_log'] = np.log(test_df1['val_4'])\ntest_df1['val_5_log'] = np.log(test_df1['val_5'])\ntest_df1['val_6_log'] = np.log(test_df1['val_6'])\ntest_df1['val_12_log_mean'] = (test_df1['val_1_log'] + test_df1['val_2_log']) / 2.0\ntest_df1['val_34_log_mean'] = (test_df1['val_3_log'] + test_df1['val_4_log']) / 2.0\ntest_df1['val_56_log_mean'] = (test_df1['val_5_log'] + test_df1['val_6_log']) / 2.0\ntest_df1['val_123_log_mean'] = (test_df1['val_1_log'] + test_df1['val_2_log'] + test_df1['val_3_log']) / 3.0\ntest_df1['val_456_log_mean'] = (test_df1['val_4_log'] + test_df1['val_5_log'] + test_df1['val_6_log']) / 3.0\n\ntest_df1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b620e4ac856a88341df626b06172a72f5c2324f"},"cell_type":"code","source":"X_test = test_df1.values\n\ny_test_pred = clf.predict(X_test)\ny_test_pred = np.reshape(y_test_pred, (y_test_pred.shape[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"763da0a3f97cc493519a210e16429e5f9d59a961"},"cell_type":"code","source":"submission_df = pd.DataFrame({'qid': test_df['qid'].values, 'prediction': y_test_pred})\nsubmission_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}