{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Add, Concatenate, Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D, Convolution2D, Reshape\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, MaxPool1D, GlobalMaxPool2D, Flatten, ZeroPadding1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad1e9fdf97f3291a7d1797b25f0c7a0c5d1f1edd"},"cell_type":"markdown","source":"## Pre-processing data\nReference: https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings"},{"metadata":{"trusted":true,"_uuid":"ba5a1b8109dee2c9fbc628d5da4a7c3447d42fb8"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.05, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 60 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"408aadcb5e49fb873035a6b6308f71794d82b0d8"},"cell_type":"code","source":"def getGloVeEmbeddings():\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    return embedding_matrix\n\ndef getFastTextEmbeddings():\n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    return embedding_matrix\n\ndef getParagramEmbeddings():\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    return embedding_matrix\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8fbc6dacdc3626b3f0b872d9e4aaed49049938ab"},"cell_type":"code","source":"glove_embedding = getGloVeEmbeddings()\nfasttext_embedding = getFastTextEmbeddings()\nparagram_embedding = getParagramEmbeddings()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ed98933bfecc2b2e75f72fd98c733a9089508d47"},"cell_type":"markdown","source":"## KMaxPooling, Folding, MGNC-CNN and MV-CNN\n\nReference: https://bicepjai.github.io/machine-learning/2017/11/10/text-class-part1.html"},{"metadata":{"trusted":true,"_uuid":"3cfab26c6cced33ef7ab84f0d36997113131d530"},"cell_type":"code","source":"from keras.engine import Layer, InputSpec\nimport tensorflow as tf\n\nclass KMaxPooling(Layer):\n    \"\"\"\n    K-max pooling layer that extracts the k-highest activations from a sequence (2nd dimension).\n    TensorFlow backend.\n    \"\"\"\n    def __init__(self, k=1, axis=1, **kwargs):\n        super(KMaxPooling, self).__init__(**kwargs)\n        self.input_spec = InputSpec(ndim=3)\n        self.k = k\n\n        assert axis in [1,2],  'expected dimensions (samples, filters, convolved_values),\\\n                   cannot fold along samples dimension or axis not in list [1,2]'\n        self.axis = axis\n\n        # need to switch the axis with the last elemnet\n        # to perform transpose for tok k elements since top_k works in last axis\n        self.transpose_perm = [0,1,2] #default\n        self.transpose_perm[self.axis] = 2\n        self.transpose_perm[2] = self.axis\n\n    def compute_output_shape(self, input_shape):\n        input_shape_list = list(input_shape)\n        input_shape_list[self.axis] = self.k\n        return tuple(input_shape_list)\n\n    def call(self, x):\n        # swap sequence dimension to get top k elements along axis=1\n        transposed_for_topk = tf.transpose(x, perm=self.transpose_perm)\n\n        # extract top_k, returns two tensors [values, indices]\n        top_k = tf.nn.top_k(transposed_for_topk, k=self.k, sorted=True, name=None)[0]\n\n        # return back to normal dimension but now sequence dimension has only k elements\n        # performing another transpose will get the tensor back to its original shape\n        # but will have k as its axis_1 size\n        transposed_back = tf.transpose(top_k, perm=self.transpose_perm)\n\n        return transposed_back\n\n\nclass Folding(Layer):\n\n    def __init__(self, **kwargs):\n        super(Folding, self).__init__(**kwargs)\n        self.input_spec = InputSpec(ndim=3)\n\n    def compute_output_shape(self, input_shape):\n        return (input_shape[0], input_shape[1], int(input_shape[2]/2))\n\n    def call(self, x):\n        input_shape = x.get_shape().as_list()\n\n        # split the tensor along dimension 2 into dimension_axis_size/2\n        # which will give us 2 tensors\n        splits = tf.split(x, num_or_size_splits=int(input_shape[2]/2), axis=2)\n\n        # reduce sums of the pair of rows we have split onto\n        reduce_sums = [tf.reduce_sum(split, axis=2) for split in splits]\n\n        # stack them up along the same axis we have reduced\n        row_reduced = tf.stack(reduce_sums, axis=2)\n        return row_reduced\n\ndef BiRNN():\n\n    inp = Input(shape=(maxlen,))\n\n    #raw      = Embedding(max_features, embed_size)(inp)\n    glove    = Embedding(max_features, embed_size, weights=[glove_embedding])(inp)\n    fast     = Embedding(max_features, embed_size, weights=[fasttext_embedding])(inp)\n    paragram = Embedding(max_features, embed_size, weights=[paragram_embedding])(inp)\n\n    x = Add()([glove, fast, paragram])\n\n    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(16, activation=\"relu\")(x)\n    x = Dropout(0.1)(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n\n    model = Model(inputs=inp, outputs=x)\n\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\n    print(model.summary())\n    \n    return model\n\ndef MGNC_CNN():\n\n    inp = Input(shape=(maxlen,))\n    \n    glove    = Embedding(max_features, embed_size, weights=[glove_embedding])(inp)\n    fast     = Embedding(max_features, embed_size, weights=[fasttext_embedding])(inp)\n    paragram = Embedding(max_features, embed_size, weights=[paragram_embedding])(inp)    \n    \n    filter_sizes = [3,5]\n    \n    conv_pools = []\n    for text_embedding in [glove,fast,paragram]:\n        for filter_size in filter_sizes:\n            l_zero = ZeroPadding1D((filter_size-1,filter_size-1))(text_embedding)\n            l_conv = Conv1D(filters=16, kernel_size=filter_size, padding='same', activation='tanh')(l_zero)\n            l_pool = GlobalMaxPool1D()(l_conv)\n            conv_pools.append(l_pool)\n            \n    l_merge = Concatenate(axis=1)(conv_pools)\n    l_dense = Dense(128, activation='relu', kernel_regularizer=regularizers.l2(0.01))(l_merge)\n    l_out = Dense(1, activation='sigmoid')(l_dense)\n        \n    model = Model(inputs=inp, outputs=l_out)\n    \n    model.compile(loss='binary_crossentropy',\n                    optimizer='adam',\n                    metrics=['acc'])\n    \n    print(model.summary())\n    \n    return model\n\ndef MV_CNN():\n    \n    inp = Input(shape=(maxlen,))\n    \n    glove    = Embedding(max_features, embed_size, weights=[glove_embedding])(inp)\n    fast     = Embedding(max_features, embed_size, weights=[fasttext_embedding])(inp)\n    paragram = Embedding(max_features, embed_size, weights=[paragram_embedding])(inp)    \n    \n    k_top = 4\n    filter_sizes = [3,5]\n\n    layer_1 = []\n    for text_embedding in [glove, fast, paragram]:\n        conv_pools = []\n        for filter_size in filter_sizes:\n            l_zero = ZeroPadding1D((filter_size-1,filter_size-1))(text_embedding)\n            l_conv = Conv1D(filters=128, kernel_size=filter_size, padding='same', activation='tanh')(l_zero)\n            l_pool = KMaxPooling(k=30, axis=1)(l_conv)\n            conv_pools.append((filter_size,l_pool))\n        layer_1.append(conv_pools)\n            \n            \n    last_layer = []\n    for layer in layer_1: # no of embeddings used\n        for (filter_size, input_feature_maps) in layer:\n            l_zero = ZeroPadding1D((filter_size-1,filter_size-1))(input_feature_maps)\n            l_conv = Conv1D(filters=128, kernel_size=filter_size, padding='same', activation='tanh')(l_zero)\n            l_pool = KMaxPooling(k=k_top, axis=1)(l_conv)\n            last_layer.append(l_pool)\n\n    l_merge = Concatenate(axis=1)(last_layer)\n    l_flat = Flatten()(l_merge)\n    l_dense = Dense(128, activation='relu')(l_flat)\n    l_out = Dense(1, activation='sigmoid')(l_dense)\n        \n    model = Model(inputs=inp, outputs=l_out)\n    \n    model.compile(loss='binary_crossentropy',\n                    optimizer='adam',\n                    metrics=['acc'])\n    \n    print(model.summary())\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b71267b19ec4b8a4c977cb3ad87b636390e5cdb9"},"cell_type":"markdown","source":"Train the model using train sample and monitor the metric on the valid sample. This is just a sample model running for 2 epochs. Changing the epochs, batch_size and model parameters might give us a better model."},{"metadata":{"trusted":true,"_uuid":"ef1e1015e7c3ab5bc5d9774e49820c4b286d7847"},"cell_type":"code","source":"#mgnccnn = MGNC_CNN()\n#mgnccnn.fit(train_X, train_y, batch_size=512, epochs=4, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a72ba82481de9f96c62c334cc40bd1d134d38e2d"},"cell_type":"markdown","source":"Now let us get the validation sample predictions and also get the best threshold for F1 score. "},{"metadata":{"trusted":true,"_uuid":"47b63dca0247a08a808db7ae6eea33065c554948"},"cell_type":"code","source":"#pred_y = mgnccnn.predict([val_X], batch_size=1024, verbose=1)\n\n#max_f1 = 0\n#max_thresh = 0\n\n#for thresh in np.arange(0.1, 0.501, 0.01):\n#    thresh = np.round(thresh, 2)\n#    score = metrics.f1_score(val_y, (pred_y > thresh).astype(int))\n#    print(\"F1 score at threshold {0} is {1}\".format(thresh, score))\n#    if score > max_f1:\n#        max_f1 = score\n#        max_thresh = thresh\n\n#print(\"** Best F1 score at threshold {0} is {1}\".format(max_thresh, max_f1))\n\n#pred_mgnccnn = pred_y","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5b784607174fdfcf76e8559a2f607f7063d8a5a"},"cell_type":"code","source":"mvcnn = MV_CNN()\nmvcnn.fit(train_X, train_y, batch_size=1024, epochs=2, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5ef71458a202ee56980983b9f7d2fecb64749f9"},"cell_type":"code","source":"pred_y = mvcnn.predict([val_X], batch_size=1024, verbose=1)\n\nmax_f1 = 0\nmax_thresh = 0\n\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    score = metrics.f1_score(val_y, (pred_y > thresh).astype(int))\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, score))\n    if score > max_f1:\n        max_f1 = score\n        max_thresh = thresh\n\nprint(\"** Best F1 score at threshold {0} is {1}\".format(max_thresh, max_f1))\n\npred_mgnccnn = pred_y","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93c2c3fc14d4281a16a9e86195c026f75143983c"},"cell_type":"markdown","source":"## Using only results from MVCNN due to time limit"},{"metadata":{"trusted":true,"_uuid":"c90fb4a4ef1b3b2ea06563a6901deac1b38822f3"},"cell_type":"code","source":"pred_y = mvcnn.predict([test_X], batch_size=1024, verbose=1)\npred_test_y = (pred_y > max_thresh).astype(int)\nout_df = pd.DataFrame({\"qid\": test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}