{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import sys, os, re, csv, codecs, numpy as np, pandas as pd\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score\n\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input,LSTM, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.layers import concatenate","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6992f4d7d290a56903011a53f6a44e0f3a15a71f"},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a73666496d93b73c5484399623a2e38adf137fa"},"cell_type":"code","source":"max_features = 95000\nmaxlen = 60 \nembed_size = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2147bbdf627009ec3e855e43ba56f2d28b393027"},"cell_type":"code","source":"X_train = train[\"question_text\"].values\ny_train = train['target'].values\nX_test = test[\"question_text\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75046a1bb9138d0fd91e3115a339ffab520746e8"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X_train)+ list(X_test))\nX_train = tokenizer.texts_to_sequences(X_train)\nX_test = tokenizer.texts_to_sequences(X_test)\nword_index = tokenizer.word_index\n\nx_train = pad_sequences(X_train, maxlen=maxlen)\nx_test = pad_sequences(X_test, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"016dfd2bf2c2e79dba2c7020f17181769b2c1386"},"cell_type":"code","source":"X_tra, X_val, y_tra, y_val = train_test_split(x_train, y_train, train_size=0.95, random_state=233)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"679edfe5aefc3c048c180f15fb40ef514392d9ab"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"191b76d7cb97b7604020c200283ad8a108346b9d"},"cell_type":"code","source":"#https://www.kaggle.com/shujian/mix-of-nn-models-based-on-meta-embedding\n\ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b86b77d6d1a42aa2c585c8fa37a4436d89de5d9e"},"cell_type":"code","source":"#embedding_matrix_1 = load_glove(word_index)\nembedding_matrix = load_fasttext(word_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0030e5a1cf3a66921b2f01cefc628db48ab4b897"},"cell_type":"code","source":"# https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#https://www.kaggle.com/yekenot/pooled-gru-fasttext\ndef get_model():\n    inp = Input(shape=(maxlen, ))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n    x = SpatialDropout1D(0.2)(x)\n    x = Bidirectional(CuDNNLSTM(80, return_sequences=True))(x)\n    atten = Attention(maxlen)(x)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    conc = concatenate([atten, avg_pool, max_pool])\n    outp = Dense(1, activation=\"sigmoid\")(conc)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f5da8e1f24525231df879ed574716a03085f3790"},"cell_type":"code","source":"from keras.layers import GRU\nmodel = get_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b57a4164aff41678c712dd4324b9f151d2f03b8"},"cell_type":"code","source":"emb_model = model.fit(X_tra, y_tra, batch_size=512, epochs=4, validation_data=(X_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d908d68b81def71b4ba5a425d1e8d6e03eed11a"},"cell_type":"code","source":"pred_val_y = model.predict([X_val], batch_size=1024, verbose=0)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    score = metrics.f1_score(y_val, (pred_val_y > thresh).astype(int))\n    best_thresh = 0\n    best_score = 0.0\n    if score > best_score:\n        best_thresh = thresh\n        best_score = score\n\nprint(\"Val F1 Score: {:.4f}\".format(best_score))\nprint(\"Val Best thresh: {:.4f}\".format(best_thresh))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6503fc5844f867d0d6fab344df3117bcb86370f5"},"cell_type":"code","source":"test_meta = np.zeros(x_test.shape[0])\npred_test_y = model.predict([x_test], batch_size=1024, verbose=1)\n\nsub = pd.DataFrame({\"qid\":test[\"qid\"].values})\nsub['prediction'] = (pred_test_y > best_thresh).astype(int)\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b849a7a77cebcb4d676dea8457ea59701af09af3"},"cell_type":"code","source":"!tail submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a3449ecd00136489b2ef8c1a1de281a21283076d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}