{"cells":[{"metadata":{"_uuid":"01f361ddc47e0b386595316fe3d7f4dabbd260db"},"cell_type":"markdown","source":"# Background\n- I just gathered all ideas."},{"metadata":{"_uuid":"802f316dc28e9f1050032e0d24ecd1b8e617158a"},"cell_type":"markdown","source":"- Thanks for various authors who help me make this kernel!\n- Below is referrence.\n- https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n-  https://www.kaggle.com/yekenot/2dcnn-textclassifier\n- https://www.kaggle.com/applecer/use-f1-to-select-model-lstm-based\n- https://www.kaggle.com/youhanlee/cnn-1d-also-can-read-sentences\n- https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n- https://www.kaggle.com/sbongo/do-pretrained-embeddings-give-you-the-extra-edge"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nfrom keras.models import Model\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import Callback\nimport warnings\n\nfrom keras.models import Sequential\nfrom keras.layers import CuDNNLSTM, Dense, Bidirectional, Dropout\nfrom keras.callbacks import Callback\nfrom sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad1e9fdf97f3291a7d1797b25f0c7a0c5d1f1edd"},"cell_type":"markdown","source":"Next steps are as follows:\n * Split the training dataset into train and val sample. Cross validation is a time consuming process and so let us do simple train val split.\n * Fill up the missing values in the text column with '_na_'\n * Tokenize the text column and convert them to vector sequences\n * Pad the sequence as needed - if the number of words in the text is greater than 'max_len' trunacate them to 'max_len' or if the number of words in the text is lesser than 'max_len' add zeros for remaining values."},{"metadata":{"trusted":true,"_uuid":"ba5a1b8109dee2c9fbc628d5da4a7c3447d42fb8"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a3e6cccc429d83fe830f8f6174eea8bc8586efb"},"cell_type":"code","source":"from keras import backend as K\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\nclass Metrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_f1s = []\n        self.val_recalls = []\n        self.val_precisions = []\n\n    def on_epoch_end(self, epoch, logs={}):\n        val_predict = (np.asarray(self.model.predict(self.model.validation_data[0]))).round()\n        val_targ = self.model.validation_data[1]\n        _val_f1 = f1_score(val_targ, val_predict)\n        _val_recall = recall_score(val_targ, val_predict)\n        _val_precision = precision_score(val_targ, val_predict)\n        self.val_f1s.append(_val_f1)\n        self.val_recalls.append(_val_recall)\n        self.val_precisions.append(_val_precision)\n        print(\" — val_f1: %f — val_precision: %f — val_recall %f\" %(_val_f1, _val_precision, _val_recall))\n        return \n \nmy_metrics = Metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7dcccf534e0d26bbba5e7a58f88d75ae4721233"},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Conv1D, Input, MaxPooling1D, Flatten, Dense, BatchNormalization, concatenate, SpatialDropout1D\nfrom keras.layers.advanced_activations import LeakyReLU\n\n# class LeakyReLU(LeakyReLU):\n#     def __init__(self, **kwargs):\n#         self.__name__ = \"LeakyReLU\"\n#         super(LeakyReLU, self).__init__(**kwargs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1be193c6eb37038654c0b020aee82ebf5f118447"},"cell_type":"code","source":"def f1_loss(y_true, y_pred):\n    \"\"\"Custom f1 loss for bicategorical\n    y must be of shape where y.shape[-1] == 2\n    y[..., 0] must be the category for true\n    y[..., 1] must be the category for false\n    \"\"\"\n    true_truth = K.dot(y_true, K.constant([1., 0.], dtype='float32', shape=(2, 1)))\n    true_false = K.dot(y_true, K.constant([0., 1.], dtype='float32', shape=(2, 1)))\n\n    y_false = K.constant(1., dtype='float32') - y_true\n\n    fake_truth = K.dot(y_false, K.constant([1., 0.], dtype='float32', shape=(2, 1)))\n    fake_false = K.dot(y_false, K.constant([0., 1.], dtype='float32', shape=(2, 1)))\n\n    TP_temp = K.sum(true_truth * y_pred)\n    FP_temp = K.sum(fake_truth * y_pred)\n    FN_temp = K.sum(fake_false * y_pred)\n\n    loss = (FP_temp + FN_temp) / (2 * TP_temp + FP_temp + FN_temp + K.epsilon())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9d263852f653e466e24f9827548d7d1a7ee7262"},"cell_type":"code","source":"!ls ../input/embeddings/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6ba4650c0ba950e9d09ace27e36a57354eb4bbd"},"cell_type":"markdown","source":"# Glove Embedding"},{"metadata":{"_uuid":"7c0010e518288bc7f588776c58610949140a139a"},"cell_type":"markdown","source":" * GoogleNews-vectors-negative300 - https://code.google.com/archive/p/word2vec/\n * glove.840B.300d - https://nlp.stanford.edu/projects/glove/\n * paragram_300_sl999 - https://cogcomp.org/page/resource_view/106\n * wiki-news-300d-1M - https://fasttext.cc/docs/en/english-vectors.html\n \n A very good explanation for different types of embeddings are given in this [kernel](https://www.kaggle.com/sbongo/do-pretrained-embeddings-give-you-the-extra-edge). Please refer the same for more details.."},{"metadata":{"trusted":true,"_uuid":"23f130e80159bb1701e449e2e91199dbfff1f1d4"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"750a3c6fd0a03f0564b7009c3a2e601635f58176"},"cell_type":"code","source":"############### CNN 1D\n\nSTRIDE_1 = 2\nSTRIDE_2 = 4\nSTRIDE_3 = 8\n\nFILTER_1 = 64\nFILTER_2 = 64\nFILTER_3 = 64\n\ninp = Input(shape=(maxlen, ))\nembed_layer1 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer1 = SpatialDropout1D(0.4)(embed_layer1)\n\n# line1 = BatchNormalization()(Input_layer)\nline1 = Conv1D(FILTER_1, STRIDE_1)(embed_layer1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1)(line1)\nline1 = Conv1D(FILTER_1, STRIDE_1)(line1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1)(line1)\nline1 = Conv1D(FILTER_1, STRIDE_1)(line1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1*2)(line1)  # global max pooling\nline1 = Flatten()(line1)\n\n# line2 = BatchNormalization()(Input_layer)\nline2 = Conv1D(FILTER_2, STRIDE_1)(embed_layer1)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1)(line2)\nline2 = Conv1D(FILTER_2, STRIDE_1)(line2)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1)(line2)\nline2 = Conv1D(FILTER_2, STRIDE_1)(line2)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1*2)(line2)  # global max pooling\nline2 = Flatten()(line2)\n\n# line3 = BatchNormalization()(Input_layer)\nline3 = Conv1D(FILTER_3, STRIDE_1)(embed_layer1)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1)(line3)\nline3 = Conv1D(FILTER_3, STRIDE_1)(line3)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1)(line3)\nline3 = Conv1D(FILTER_3, STRIDE_1)(line3)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1*2)(line3)  # global max pooling\nline3 = Flatten()(line3)\n\nconcat_layer = concatenate([line1, line2, line3])\n\nconv1d_dense = Dense(1024, activation='relu')(concat_layer)\n\n\n\n############### RNN\nembed_layer2 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer2 = SpatialDropout1D(0.4)(embed_layer2)\n\nrnn_line = Bidirectional(CuDNNLSTM(64, return_sequences=True), input_shape=(maxlen, embed_size))(embed_layer2)\nrnn_line = Bidirectional(CuDNNLSTM(64))(rnn_line)\nrnn_dense = Dense(1024, activation='relu')(rnn_line)\n\n\n\n############### CNN 2D\n\nfilter_sizes = [1,2,3,5]\nnum_filters = 36\n\nembed_layer3 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer3 = SpatialDropout1D(0.4)(embed_layer3)\n\nx = SpatialDropout1D(0.4)(embed_layer3)\nx = Reshape((maxlen, embed_size, 1))(x)\n\nconv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embed_size),\n                             kernel_initializer='he_normal', activation='elu')(x)\nconv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embed_size),\n                             kernel_initializer='he_normal', activation='elu')(x)\nconv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embed_size), \n                             kernel_initializer='he_normal', activation='elu')(x)\nconv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embed_size),\n                             kernel_initializer='he_normal', activation='elu')(x)\n\nmaxpool_0 = MaxPool2D(pool_size=(maxlen - filter_sizes[0] + 1, 1))(conv_0)\nmaxpool_1 = MaxPool2D(pool_size=(maxlen - filter_sizes[1] + 1, 1))(conv_1)\nmaxpool_2 = MaxPool2D(pool_size=(maxlen - filter_sizes[2] + 1, 1))(conv_2)\nmaxpool_3 = MaxPool2D(pool_size=(maxlen - filter_sizes[3] + 1, 1))(conv_3)\n\ncnn2d_dense = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])   \ncnn2d_dense = Flatten()(cnn2d_dense)\ncnn2d_dense = Dropout(0.1)(cnn2d_dense)\n\ntotal = concatenate([cnn2d_dense, conv1d_dense, rnn_dense])\n\npreds = Dense(1, activation='sigmoid')(total)\n\nmodel = Model(inputs=inp, outputs=preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2653e05e44a4603843b43a89f768e0b1d6530798"},"cell_type":"code","source":"from keras.utils import plot_model\nplot_model(model, to_file='model.png')\n\nfrom IPython.display import SVG\nfrom keras.utils.vis_utils import model_to_dot\n\nSVG(model_to_dot(model).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa4ebb36b5bed60dc4db2e94a6bd758f47c54768"},"cell_type":"code","source":"model.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a560ab0dbab9cf6fdbdae6721ec030e300f19d78"},"cell_type":"code","source":"## Train the model \nmodel.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff43855164472de035a5a1d80b3db4838684701a"},"cell_type":"code","source":"pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2a33c252f31fddcc65896053184226128562776"},"cell_type":"markdown","source":"Results seem to be better than the model without pretrained embeddings."},{"metadata":{"trusted":true,"_uuid":"d51ff8ed6a87b488fec3ac84ca50df661d7c8193"},"cell_type":"code","source":"pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39d4fedab4ac170863a0ee1ca3aa9be1ee58fe02"},"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"62778988128f5c55397800620ba16bfa536b11cd"},"cell_type":"markdown","source":"# Wiki News FastText Embeddings"},{"metadata":{"trusted":true,"_uuid":"6f3d0fd28dd2b04eaccb732b96b872e5a223d962"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"176a13f1d019e46e1c881296404bacac6ee4e46e"},"cell_type":"code","source":"############### CNN 1D\n\nSTRIDE_1 = 2\nSTRIDE_2 = 4\nSTRIDE_3 = 8\n\nFILTER_1 = 64\nFILTER_2 = 64\nFILTER_3 = 64\n\ninp = Input(shape=(maxlen, ))\nembed_layer1 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer1 = SpatialDropout1D(0.4)(embed_layer1)\n\n# line1 = BatchNormalization()(Input_layer)\nline1 = Conv1D(FILTER_1, STRIDE_1)(embed_layer1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1)(line1)\nline1 = Conv1D(FILTER_1, STRIDE_1)(line1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1)(line1)\nline1 = Conv1D(FILTER_1, STRIDE_1)(line1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1*2)(line1)  # global max pooling\nline1 = Flatten()(line1)\n\n# line2 = BatchNormalization()(Input_layer)\nline2 = Conv1D(FILTER_2, STRIDE_1)(embed_layer1)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1)(line2)\nline2 = Conv1D(FILTER_2, STRIDE_1)(line2)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1)(line2)\nline2 = Conv1D(FILTER_2, STRIDE_1)(line2)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1*2)(line2)  # global max pooling\nline2 = Flatten()(line2)\n\n# line3 = BatchNormalization()(Input_layer)\nline3 = Conv1D(FILTER_3, STRIDE_1)(embed_layer1)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1)(line3)\nline3 = Conv1D(FILTER_3, STRIDE_1)(line3)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1)(line3)\nline3 = Conv1D(FILTER_3, STRIDE_1)(line3)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1*2)(line3)  # global max pooling\nline3 = Flatten()(line3)\n\nconcat_layer = concatenate([line1, line2, line3])\n\nconv1d_dense = Dense(1024, activation='relu')(concat_layer)\n\n\n\n############### RNN\nembed_layer2 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer2 = SpatialDropout1D(0.4)(embed_layer2)\n\nrnn_line = Bidirectional(CuDNNLSTM(64, return_sequences=True), input_shape=(maxlen, embed_size))(embed_layer2)\nrnn_line = Bidirectional(CuDNNLSTM(64))(rnn_line)\nrnn_dense = Dense(1024, activation='relu')(rnn_line)\n\n\n\n############### CNN 2D\n\nfilter_sizes = [1,2,3,5]\nnum_filters = 36\n\nembed_layer3 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer3 = SpatialDropout1D(0.4)(embed_layer3)\n\nx = SpatialDropout1D(0.4)(embed_layer3)\nx = Reshape((maxlen, embed_size, 1))(x)\n\nconv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embed_size),\n                             kernel_initializer='he_normal', activation='elu')(x)\nconv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embed_size),\n                             kernel_initializer='he_normal', activation='elu')(x)\nconv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embed_size), \n                             kernel_initializer='he_normal', activation='elu')(x)\nconv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embed_size),\n                             kernel_initializer='he_normal', activation='elu')(x)\n\nmaxpool_0 = MaxPool2D(pool_size=(maxlen - filter_sizes[0] + 1, 1))(conv_0)\nmaxpool_1 = MaxPool2D(pool_size=(maxlen - filter_sizes[1] + 1, 1))(conv_1)\nmaxpool_2 = MaxPool2D(pool_size=(maxlen - filter_sizes[2] + 1, 1))(conv_2)\nmaxpool_3 = MaxPool2D(pool_size=(maxlen - filter_sizes[3] + 1, 1))(conv_3)\n\ncnn2d_dense = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])   \ncnn2d_dense = Flatten()(cnn2d_dense)\ncnn2d_dense = Dropout(0.1)(cnn2d_dense)\n\ntotal = concatenate([cnn2d_dense, conv1d_dense, rnn_dense])\n\npreds = Dense(1, activation='sigmoid')(total)\n\nmodel = Model(inputs=inp, outputs=preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"890575acaf51e887bf784af5abafd9ff7f3abe8a"},"cell_type":"code","source":"model.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1001f593779021dcefe5a9266825c0f35da33290"},"cell_type":"code","source":"## Train the model \nmodel.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7ab4100f723ad535528865b1edc7896bce80223"},"cell_type":"code","source":"pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_fasttext_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3216362afb0f49579d287a06f13adf8cd7d8b0cf"},"cell_type":"code","source":"pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f24f9753ff1d933fa4f75a0ba34df305632d6e93"},"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ca44ac68bf404b9c26e07fbcc9c8ac793e04510"},"cell_type":"markdown","source":"# Paragram Embeddings"},{"metadata":{"trusted":true,"_uuid":"25ec1aac4aedbf431a2d30de64030ce8e3203c18"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d36fa773d52e91e7f80142acb372e1cfd06061a4"},"cell_type":"code","source":"############### CNN 1D\n\nSTRIDE_1 = 2\nSTRIDE_2 = 4\nSTRIDE_3 = 8\n\nFILTER_1 = 64\nFILTER_2 = 64\nFILTER_3 = 64\n\ninp = Input(shape=(maxlen, ))\nembed_layer1 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer1 = SpatialDropout1D(0.4)(embed_layer1)\n\n# line1 = BatchNormalization()(Input_layer)\nline1 = Conv1D(FILTER_1, STRIDE_1)(embed_layer1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1)(line1)\nline1 = Conv1D(FILTER_1, STRIDE_1)(line1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1)(line1)\nline1 = Conv1D(FILTER_1, STRIDE_1)(line1)\nline1 = Activation(LeakyReLU())(line1)\nline1 = MaxPooling1D(STRIDE_1*2)(line1)  # global max pooling\nline1 = Flatten()(line1)\n\n# line2 = BatchNormalization()(Input_layer)\nline2 = Conv1D(FILTER_2, STRIDE_1)(embed_layer1)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1)(line2)\nline2 = Conv1D(FILTER_2, STRIDE_1)(line2)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1)(line2)\nline2 = Conv1D(FILTER_2, STRIDE_1)(line2)\nline2 = Activation(LeakyReLU())(line2)\nline2 = MaxPooling1D(STRIDE_1*2)(line2)  # global max pooling\nline2 = Flatten()(line2)\n\n# line3 = BatchNormalization()(Input_layer)\nline3 = Conv1D(FILTER_3, STRIDE_1)(embed_layer1)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1)(line3)\nline3 = Conv1D(FILTER_3, STRIDE_1)(line3)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1)(line3)\nline3 = Conv1D(FILTER_3, STRIDE_1)(line3)\nline3 = Activation(LeakyReLU())(line3)\nline3 = MaxPooling1D(STRIDE_1*2)(line3)  # global max pooling\nline3 = Flatten()(line3)\n\nconcat_layer = concatenate([line1, line2, line3])\n\nconv1d_dense = Dense(1024, activation='relu')(concat_layer)\n\n\n\n############### RNN\nembed_layer2 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer2 = SpatialDropout1D(0.4)(embed_layer2)\n\nrnn_line = Bidirectional(CuDNNLSTM(64, return_sequences=True), input_shape=(maxlen, embed_size))(embed_layer2)\nrnn_line = Bidirectional(CuDNNLSTM(64))(rnn_line)\nrnn_dense = Dense(1024, activation='relu')(rnn_line)\n\n\n\n############### CNN 2D\n\nfilter_sizes = [1,2,3,5]\nnum_filters = 36\n\nembed_layer3 = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nembed_layer3 = SpatialDropout1D(0.4)(embed_layer3)\n\nx = SpatialDropout1D(0.4)(embed_layer3)\nx = Reshape((maxlen, embed_size, 1))(x)\n\nconv_0 = Conv2D(num_filters, kernel_size=(filter_sizes[0], embed_size),\n                             kernel_initializer='he_normal', activation='elu')(x)\nconv_1 = Conv2D(num_filters, kernel_size=(filter_sizes[1], embed_size),\n                             kernel_initializer='he_normal', activation='elu')(x)\nconv_2 = Conv2D(num_filters, kernel_size=(filter_sizes[2], embed_size), \n                             kernel_initializer='he_normal', activation='elu')(x)\nconv_3 = Conv2D(num_filters, kernel_size=(filter_sizes[3], embed_size),\n                             kernel_initializer='he_normal', activation='elu')(x)\n\nmaxpool_0 = MaxPool2D(pool_size=(maxlen - filter_sizes[0] + 1, 1))(conv_0)\nmaxpool_1 = MaxPool2D(pool_size=(maxlen - filter_sizes[1] + 1, 1))(conv_1)\nmaxpool_2 = MaxPool2D(pool_size=(maxlen - filter_sizes[2] + 1, 1))(conv_2)\nmaxpool_3 = MaxPool2D(pool_size=(maxlen - filter_sizes[3] + 1, 1))(conv_3)\n\ncnn2d_dense = Concatenate(axis=1)([maxpool_0, maxpool_1, maxpool_2, maxpool_3])   \ncnn2d_dense = Flatten()(cnn2d_dense)\ncnn2d_dense = Dropout(0.1)(cnn2d_dense)\n\ntotal = concatenate([cnn2d_dense, conv1d_dense, rnn_dense])\n\npreds = Dense(1, activation='sigmoid')(total)\n\nmodel = Model(inputs=inp, outputs=preds)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b15cff18d5f604f3c38f2f5119824843fc487e1"},"cell_type":"code","source":"model.compile(loss='binary_crossentropy',\n              optimizer='adam',\n              metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65cdf51e36b60c5733924706e0f1c4b17cd8364a"},"cell_type":"code","source":"## Train the model \nmodel.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9abdfd1cf15257f2c0c2181a13327796e8d4584e"},"cell_type":"code","source":"pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_paragram_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"99cb9f6145da909bd7436e46d47547efc097499d"},"cell_type":"code","source":"pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1312b7a4c3b67ca4ebd26fb083dbac3b6635dc2"},"cell_type":"markdown","source":"# Blend"},{"metadata":{"trusted":true,"_uuid":"449bc59fdc9a719aa0759ac51a4481df113604ca"},"cell_type":"code","source":"pred_val_y = 0.33*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.34*pred_paragram_val_y \nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4fdbeffc0f84643d2832eec49234bd9d6c6e216b"},"cell_type":"markdown","source":"The result seems to better than individual pre-trained models and so we let us create a submission file using this model blend."},{"metadata":{"trusted":true,"_uuid":"c90fb4a4ef1b3b2ea06563a6901deac1b38822f3"},"cell_type":"code","source":"pred_test_y = 0.33*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.34*pred_paragram_test_y\npred_test_y = (pred_test_y>0.35).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51f340130ff6d52cfb6545a1f15c6d7d8e6a1db3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}