{"cells":[{"metadata":{"_uuid":"cc406dac1c0854793fe8672741f97f773f829909"},"cell_type":"markdown","source":"##### * Based on Shujian Liu's kernel: https://www.kaggle.com/shujian/blend-of-lstm-and-cnn-with-4-embeddings-1200d\n* Dense layer just after the embedding layer to reduce dimension(600 to 128).\n* Pretrained embeddings: trainable = False\n\n* Based on SRK's kernel: https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* Concatenate embeddings instead of blending predictions. Added SpatialDropout1D.\n* Added word2vec from https://www.kaggle.com/strideradu/word2vec-and-gensim-go-go-go.\n* Modified the code to choose best threshold"},{"metadata":{"trusted":true,"_uuid":"144fe5891359c058bba89c6bdef4054d3f324f34"},"cell_type":"code","source":"## Settings:\n# some config values \nmax_features = 90000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 50 # max number of words in a question to use\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport tensorflow as tf\nimport numpy as np # linear algebra\nimport random\nimport os \nos.environ['PYTHONHASHSEED'] = '11'\nnp.random.seed(22)\nrandom.seed(33)\ntf.set_random_seed(44)\n\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nimport gc\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, Embedding, CuDNNGRU\nfrom keras.layers import Bidirectional, GlobalMaxPooling1D\nfrom keras.models import Model\nfrom keras.optimizers import Adam\nfrom keras.initializers import glorot_uniform\nfrom keras.callbacks import Callback\nfrom keras.models import clone_model\nimport keras.backend as K","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc1f67d63083cc7cbe426a288a808e9d886fd97a"},"cell_type":"code","source":"t0 = time.time()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09b81ba9a82704f678f0c0cde5d10d7ab80e719b"},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be96ce6e199e200faa34a53635ac2b195a99a057"},"cell_type":"code","source":"train_df[\"question_text\"] = train_df[\"question_text\"].str.lower()\ntest_df[\"question_text\"] = test_df[\"question_text\"].str.lower()\n    \ntrain_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: clean_text(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ba5a1b8109dee2c9fbc628d5da4a7c3447d42fb8"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrunc = 'pre'\ntrain_X = pad_sequences(train_X, maxlen=maxlen, truncating=trunc)\nval_X = pad_sequences(val_X, maxlen=maxlen, truncating=trunc)\ntest_X = pad_sequences(test_X, maxlen=maxlen, truncating=trunc)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"717f7dcd5ccf71e83d0f062e221c46db39845e4d"},"cell_type":"markdown","source":"**Glove Embeddings:**"},{"metadata":{"trusted":true,"_uuid":"23f130e80159bb1701e449e2e91199dbfff1f1d4"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_1 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector\n\ndel embeddings_index; gc.collect() ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bc6bab22dd12a09378f4b8b159cb7a5d88a3e7c0"},"cell_type":"markdown","source":"**Wiki News FastText Embeddings:**"},{"metadata":{"trusted":true,"_uuid":"6f3d0fd28dd2b04eaccb732b96b872e5a223d962"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_2 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_2[i] = embedding_vector\n        \ndel embeddings_index; gc.collect()         ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ca44ac68bf404b9c26e07fbcc9c8ac793e04510"},"cell_type":"markdown","source":"**Paragram Embeddings:**"},{"metadata":{"trusted":true,"_uuid":"25ec1aac4aedbf431a2d30de64030ce8e3203c18"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_3 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_3[i] = embedding_vector\n\ndel embeddings_index; gc.collect()         ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4474db5a07969ca0703c65c3f95c2b0f61c1468b"},"cell_type":"markdown","source":"**Word2vec Embeddings:**"},{"metadata":{"trusted":true,"_uuid":"9b83944a5ad66f4b5d60d57bed12b9c19035a19a"},"cell_type":"code","source":"# https://www.kaggle.com/strideradu/word2vec-and-gensim-go-go-go\nfrom gensim.models import KeyedVectors\n\nEMBEDDING_FILE = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_4 = (np.random.rand(nb_words, embed_size) - 0.5) / 5.0\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    if word in embeddings_index:\n        embedding_vector = embeddings_index.get_vector(word)\n        embedding_matrix_4[i] = embedding_vector\n        \ndel embeddings_index; gc.collect()         ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"94e1be128208411c4d95a26610a4d58e66be013c"},"cell_type":"markdown","source":"** Combine :**"},{"metadata":{"trusted":true,"_uuid":"957c8187e76769e3165e900a9784fc4bb30ffca4"},"cell_type":"code","source":"embedding_matrix = np.concatenate((embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4), axis=1)  \ndel embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4\ngc.collect()\nnp.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"250734f80a9799ebbe3c99dbbc16c65f599ad4ff"},"cell_type":"code","source":"print(f'Done preprocessing {time.time() - t0:.1f}s')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfd358ac39ca62c6037973ad591af60362628262"},"cell_type":"code","source":"class ExponentialMovingAverage(Callback):\n    def __init__(self, model, decay=0.999, mode='epoch', n=100):\n        \"\"\"\n        mode: 'epoch': Do update_weights every epoch.\n              'batch':                   every n batches.\n        n   :\n        \"\"\"\n        self.decay = decay\n        self.mode = mode\n        self.ema_model = clone_model(model)\n        self.ema_model.set_weights(model.get_weights())\n        self.n = n\n        if self.mode is 'batch':\n            self.cnt = 0\n        self.ema_weights = [K.get_value(w) for w in model.trainable_weights]\n        self.n_weights = len(self.ema_weights)\n        super(ExponentialMovingAverage, self).__init__()\n\n    def on_batch_end(self, batch, logs={}):\n        if self.mode is 'batch':\n            self.cnt += 1\n            if self.cnt % self.n == 0:\n                self.update_weights()\n\n    def on_epoch_end(self, epoch, logs={}):\n        if self.mode is 'epoch':\n            self.update_weights()\n        for var, w in zip(self.ema_model.trainable_weights, self.ema_weights):\n            K.set_value(var, w)\n\n    def update_weights(self):\n        for w_old, var_new in zip(self.ema_weights, self.model.trainable_weights):\n            w_old += (1 - self.decay) * (K.get_value(var_new) - w_old)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"77c68caa1c640b9e463a196cbca50304ae134d75"},"cell_type":"markdown","source":"**GRU:**"},{"metadata":{"trusted":true,"_uuid":"189e46038828bb0facf8a6e7c85582710db36dce"},"cell_type":"code","source":"def create_rnn_model(rnn, maxlen, embedding, max_features, embed_size,\n                     rnn_dim=64, dense1_dim=100, dense2_dim=50,\n                     embed_trainable=False, seed=123):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding],\n                  trainable=embed_trainable)(inp)\n    x = Dense(dense1_dim, activation='relu',\n              kernel_initializer=glorot_uniform(seed=seed))(x)\n    x = Bidirectional(rnn(rnn_dim, return_sequences=True,\n                          kernel_initializer=glorot_uniform(seed=seed)))(x)\n    x = GlobalMaxPooling1D()(x)\n    x = Dense(dense2_dim, activation='relu',\n              kernel_initializer=glorot_uniform(seed=seed))(x)\n    x = Dense(1, activation='sigmoid',\n              kernel_initializer=glorot_uniform(seed=seed))(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam')\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f71688f05a00613d2df4a9f009d2b56e02d22a8"},"cell_type":"code","source":"def f1_best(y_val, pred_val):\n    best_f1 = 0\n    best_thresh = 0\n    for thresh in np.linspace(0.2, 0.4, 41):\n        f1 = metrics.f1_score(y_val, (pred_val > thresh).astype(int))\n        if f1 > best_f1:\n            best_f1 = f1\n            best_thresh = thresh\n    return best_f1, best_thresh\n\nembed_ids = [list(range(300)), list(range(300, 600)),\n             list(range(600, 900)), list(range(900, 1200))]\nembed_ids_dict = {1: [embed_ids[0], embed_ids[1], embed_ids[2], embed_ids[3]],\n                  2: [embed_ids[0] + embed_ids[1],\n                      embed_ids[0] + embed_ids[2],\n                      embed_ids[0] + embed_ids[3],\n                      embed_ids[1] + embed_ids[2],\n                      embed_ids[1] + embed_ids[3],\n                      embed_ids[2] + embed_ids[3]],\n                  3: [embed_ids[0] + embed_ids[1] + embed_ids[2],\n                      embed_ids[0] + embed_ids[1] + embed_ids[3],\n                      embed_ids[0] + embed_ids[2] + embed_ids[3],\n                      embed_ids[1] + embed_ids[2] + embed_ids[3]],\n                  4: [embed_ids[0] + embed_ids[1] + embed_ids[2] + embed_ids[3]]}\nembed_ids_lst = embed_ids_dict[2]\nembed_size = 600\n\nrnn = CuDNNGRU\nembed_trainable = False\n\nn_models = 6\nepochs = 7\nbatch_size = 512\ndense1_dim = rnn_dim = 128\ndense2_dim = 2 * rnn_dim\n\nema_n = int(len(train_y) / batch_size / 10)\ndecay = 0.9\nscores = []\n\npred_avg = np.zeros((len(val_y), 1))\npred_test_avg = np.zeros((test_df.shape[0], 1))\nfor i in range(n_models):\n    t1 = time.time()\n    seed = 101 + 11 * i\n    cols_in_use = embed_ids_lst[i % len(embed_ids_lst)]\n    model = create_rnn_model(rnn, maxlen, embedding_matrix[:, cols_in_use],\n                             max_features, embed_size,\n                             rnn_dim=rnn_dim,\n                             dense1_dim=dense1_dim,\n                             dense2_dim=dense2_dim,\n                             embed_trainable=embed_trainable,\n                             seed=seed)\n    ema = ExponentialMovingAverage(model, decay=decay, mode='batch', n=ema_n)\n    model.fit(train_X, train_y, batch_size=batch_size, epochs=epochs,\n              callbacks=[ema], verbose=0)\n    m = ema.ema_model\n    t_per_epoch = (time.time() - t1) / epochs\n    pred = m.predict([val_X])\n    pred_avg += pred\n    pred_test = m.predict([test_X])\n    pred_test_avg += pred_test\n    f1_one, thresh_one = f1_best(val_y, pred)\n    f1_avg, thresh_avg = f1_best(val_y, pred_avg / (i + 1))\n    nll_one = metrics.log_loss(val_y, pred)\n    nll_avg = metrics.log_loss(val_y, pred_avg / (i + 1))\n    auc_one = metrics.roc_auc_score(val_y, pred)\n    auc_avg = metrics.roc_auc_score(val_y, pred_avg)\n    print(f'  n_model:{i + 1} epoch:{epochs} F1:{f1_avg:.4f} th:{thresh_avg:.3f} ' +\n          f'AUC:{auc_avg:.4f} NLL:{nll_avg:.4f} One:{f1_one:.4f} {auc_one:.4f} {nll_one:.4f} ' +\n          f'Time:{time.time() - t1:.1f}s  {t_per_epoch:.1f}s/epoch')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"151ee97c9f1aa3039c68c96c95efcadad9211437"},"cell_type":"code","source":"pred_test_avg /= n_models\npred_test_avg = (pred_test_avg>thresh_avg).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_avg\nout_df.to_csv(\"submission.csv\", index=False)\n\nprint(f'Done:{time.time() - t0:.1f}s')","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}