{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n\n(and other links in notebook)\n\nRemark:\nmodel overfits like hell...\n\nv6.1:\nincreased size of conv from 32 -> 100\nFor commit I have to disable training and tuning stage and fit on whole model, otherwise the running time is longer than 2 hours."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    print('\\nbest threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77b7c5199c59943744495e62d7c0f73f68769e17"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf[\"question_text\"].fillna(\"_##_\",inplace=True)\nmax_len = df['question_text'].apply(lambda x:len(x)).max()\nprint('max length of sequences:',max_len)\n# df = df.sample(frac=0.1)\n\nprint('columns:',df.columns)\npd.set_option('display.max_columns',None)\nprint('df head:',df.head())\nprint('example of the question text values:',df['question_text'].head().values)\nprint('what values contains target:',df.target.unique())\n\nprint('Computing class weights....')\n#https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(df.target.values),\n                                                 df.target.values)\nprint('class_weights:',class_weights)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f41b23c1f3f4eed0d8d419974fe795b63f3df50b"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#dim of vectors\ndim = 300\n# max words in vocab\nnum_words = 50000\n# max number in questions\nmax_len = 100 \n\nprint('Fiting tokenizer')\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=num_words)\ntokenizer.fit_on_texts(df['question_text'])\n\nprint('spliting data')\ndf_train,df_test = train_test_split(df, random_state=1)\n\nprint('text to sequence')\nx_train = tokenizer.texts_to_sequences(df_train['question_text'])\nx_test = tokenizer.texts_to_sequences(df_test['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx_train = pad_sequences(x_train,maxlen=max_len)\nx_test = pad_sequences(x_test, maxlen=max_len)\n\n## Get the target values\ny_train = df_train['target'].values\ny_test = df_test['target'].values\n\nprint(x_train.shape)\nprint(y_train.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db71c4ce3c1e3743f964c1a6e43a11644ee53cb4","scrolled":true},"cell_type":"code","source":"# https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\nprint('loading word2vec model...')\nword2vec = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)\nprint('vocab:',len(word2vec.vocab))\n\nall_embs = word2vec.vectors\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(emb_mean,emb_std)\n\nprint(num_words,' from ',len(tokenizer.word_index.items()))\n# num_words = min(num_words, len(tokenizer.word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (num_words, dim))\n\n# embedding_matrix = np.zeros((num_words, dim))\ncount = 0\nfor word, i in tokenizer.word_index.items():\n    if i>=num_words:\n        break\n    if word in word2vec.vocab:\n        embedding_matrix[i] = word2vec.word_vec(word)\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix.shape)\nprint('Number of words not in vocab:',count)\n\ndel word2vec\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d6abe3bd0a68cf06bc006b9b6efd799b90c7dfb"},"cell_type":"code","source":"print('glove...')\n\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt'))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n\nword_index = tokenizer.word_index\n# num_words = min(num_words, len(word_index))\nembedding_matrix_glov = np.random.normal(emb_mean, emb_std, (num_words, dim))\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_glov[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e78bb6fa9a5f4c8bdc6d9fd09534076276bc703e"},"cell_type":"code","source":"print('paragram...')\n\nEMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n\nword_index = tokenizer.word_index\n# num_words = min(num_words, len(word_index))\nembedding_matrix_para = np.random.normal(emb_mean, emb_std, (num_words, dim))\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_para[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"504f0213725bb0ffc728c58fa0845b35c542179c"},"cell_type":"code","source":"print('wiki news...')\n\nEMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\n\nprint(len(all_embs))\n\nword_index = tokenizer.word_index\nembedding_matrix_wiki = np.random.normal(emb_mean, emb_std, (num_words, dim))\n\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_wiki[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_wiki.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55cb2ac1dca7de9fba51e8a7e5dba402159be302"},"cell_type":"code","source":"from keras.layers import Dense, Input,Embedding, Dropout, Activation, CuDNNLSTM,BatchNormalization,concatenate\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Concatenate, GlobalAveragePooling1D,Average,Conv1D,GlobalMaxPooling1D\nfrom keras.models import Model\nfrom keras.callbacks import Callback,EarlyStopping\nfrom keras.engine import Layer\nfrom keras.initializers import Ones, Zeros\nimport keras.backend as K\nfrom keras import regularizers\nfrom keras import constraints\n\nclass CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())\n\n# https://arxiv.org/abs/1607.06450\n# https://github.com/keras-team/keras/issues/3878\nclass LayerNormalization(Layer):\n    def __init__(self, **kwargs):\n        super(LayerNormalization, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.gain = self.add_weight(name='gain', shape=input_shape[-1:],\n                                    initializer=Ones(), trainable=True)\n        self.bias = self.add_weight(name='bias', shape=input_shape[-1:],\n                                    initializer=Zeros(), trainable=True)\n        super(LayerNormalization, self).build(input_shape)\n\n    def call(self, x, **kwargs):\n        mean = K.mean(x, axis=-1, keepdims=True)\n        std = K.std(x, axis=-1, keepdims=True)\n        # dot = *\n        # std+eps because of possible nans..\n        return self.gain * (x - mean) / (std + K.epsilon()) + self.bias\n\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\n# Yoon Kim's shallow cnn model: https://arxiv.org/pdf/1408.5882.pdf\n# https://github.com/raghakot/keras-text/blob/master/keras_text/models/sequence_encoders.py\n\n#word2vec    \ninp1 = Input(shape=(max_len,))\ne1 = Embedding(num_words, dim, weights=[embedding_matrix])(inp1)\n\n#glove\ninp2 = Input(shape=(max_len,))\ne2 = Embedding(num_words, dim, weights=[embedding_matrix_glov])(inp2)\n\n#wiki\ninp3 = Input(shape=(max_len,))\ne3 = Embedding(num_words, dim, weights=[embedding_matrix_wiki])(inp3)\n\n#para\ninp4 = Input(shape=(max_len,))\ne4 = Embedding(num_words, dim, weights=[embedding_matrix_para])(inp4)\n\n#no pretriained\ninp5 = Input(shape=(max_len,))\ne5 = Embedding(num_words, dim,)(inp5)\n\nx = Concatenate()([e1,e2,e3,e4,e5])\npooled_tensors = []\nkernel_sizes = [3, 4, 5]\nfor kernel_size in kernel_sizes:\n    l1 = Conv1D(100, kernel_size=kernel_size, activation='relu', )(x)\n    l1 = GlobalMaxPooling1D()(l1)\n    pooled_tensors.append(l1)\n\nif len(kernel_sizes) > 1:\n    x = concatenate(pooled_tensors, axis=-1)\nelse:\n    x = pooled_tensors[0]\n\nx = Dense(1, activation='sigmoid')(x)\n\nmodel = Model(inputs=[inp1,inp2, inp3, inp4,inp5], outputs=x)\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nprint(model.summary())\n\n# clr = CyclicLR(base_lr=0.001, max_lr=0.007,\n#                         step_size=300., mode='exp_range',\n#                         gamma=0.99994)\n# clr = CyclicLR()\n\n\n# # for commiting the model to competition i need to comment these sections....otherwise the running time will be more then 2h on gpu...\n# history = model.fit([x_train,x_train,x_train,x_train,x_train],y_train, \n#                       batch_size=512, \n#                       validation_split=0.2,\n#                       epochs=100,\n#                       #overfits rather soon\n#                       callbacks=[EarlyStopping(patience=2)])\n\n# print('training done....')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da921fc2d216b1e2db6819ea13ab0b0c4a0d731d"},"cell_type":"code","source":"# _, ax = plt.subplots(1, 2, figsize=(12, 6))\n# ax[0].plot(history.history['loss'], label='loss')\n# ax[0].plot(history.history['val_loss'], label='val_loss')\n# ax[0].legend()\n# ax[0].set_title('loss')\n\n# ax[1].plot(history.history['acc'], label='acc')\n# ax[1].plot(history.history['val_acc'], label='val_acc')\n# ax[1].legend()\n# ax[1].set_title('acc')\n\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cde59508feb4a5c5f398128f9a5176e9a1912236","scrolled":true},"cell_type":"code","source":"# #for train set\n# y_pred = model.predict([x_train,x_train,x_train,x_train,x_train],batch_size=1024, verbose=1)\n# search_result = threshold_search(y_train, y_pred)\n# print(search_result)\n# y_pred = y_pred>search_result['threshold']\n# y_pred = y_pred.astype(int)\n\n# print('RESULTS ON TRAINING SET:\\n',classification_report(y_train,y_pred))\n\n\n# #for test set\n# y_pred = model.predict([x_test,x_test,x_test,x_test,x_test],batch_size=1024, verbose=1)\n# search_result = threshold_search(y_test, y_pred)\n# print(search_result)\n# y_pred = y_pred>search_result['threshold']\n# y_pred = y_pred.astype(int)\n\n# print('RESULTS ON TEST SET:\\n',classification_report(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43c8739f3913e5785ce8b73179dd0bb695f50a4e"},"cell_type":"markdown","source":"   # training \n       Train on 783672 samples, validate on 195919 samples\n    Epoch 1/100\n    783672/783672 [==============================] - 781s 996us/step - loss: 0.1180 - acc: 0.9549 - val_loss: 0.1069 - val_acc: 0.9569\n    Epoch 2/100\n    783672/783672 [==============================] - 770s 983us/step - loss: 0.0860 - acc: 0.9662 - val_loss: 0.1123 - val_acc: 0.9566\n    Epoch 3/100\n    783672/783672 [==============================] - 770s 983us/step - loss: 0.0543 - acc: 0.9795 - val_loss: 0.1342 - val_acc: 0.9544\n    training done....\n\n# results \n\n       979591/979591 [==============================] - 209s 213us/step\n        threshold = 0.990000 | score = 0.329381\n        best threshold is  0.360000 with score 0.875895\n        {'threshold': 0.36, 'f1': 0.8758945511433612}\n        RESULTS ON TRAINING SET:\n                      precision    recall  f1-score   support\n\n                  0       0.99      0.99      0.99    919150\n                  1       0.88      0.87      0.88     60441\n\n        avg / total       0.98      0.98      0.98    979591\n\n        326531/326531 [==============================] - 70s 214us/step\n        threshold = 0.990000 | score = 0.131481\n        best threshold is  0.220000 with score 0.619031\n        {'threshold': 0.22, 'f1': 0.6190313546007241}\n        RESULTS ON TEST SET:\n                      precision    recall  f1-score   support\n\n                  0       0.98      0.97      0.97    306162\n                  1       0.58      0.67      0.62     20369\n\n        avg / total       0.95      0.95      0.95    326531"},{"metadata":{"trusted":true,"_uuid":"3beb75a2101c603f182934220444bf573647220c"},"cell_type":"code","source":"#fit final model on all data\nprint('text to sequence')\nx = tokenizer.texts_to_sequences(df['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx = pad_sequences(x,maxlen=max_len)\n\n## Get the target values\ny = df['target'].values\n\nprint('fiting final model...')\nhistory = model.fit([x,x,x,x,x],y, batch_size=512, epochs=4)\n\nprint('fitting on full data done...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49e1bd5b69b428e57ad8248868d4b23c9c6f9d84"},"cell_type":"code","source":"_, ax = plt.subplots(1, 2, figsize=(12, 6))\nax[0].plot(history.history['loss'], label='loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['acc'], label='acc')\nax[1].legend()\nax[1].set_title('acc')\n\nplt.show()\n\ny_pred = model.predict([x,x,x,x,x],batch_size=1024, verbose=1)\nsearch_result = threshold_search(y, y_pred)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint(classification_report(y,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"#submission\nprint('Loading test data...')\ndf_final = pd.read_csv('../input/test.csv')\ndf_final[\"question_text\"].fillna(\"_##_\", inplace=True)\n\nx_final=tokenizer.texts_to_sequences(df_final['question_text'])\nx_final = pad_sequences(x_final,maxlen=max_len)\n\ny_pred = model.predict([x_final,x_final,x_final,x_final,x_final],batch_size=1024,verbose=1)\ny_pred = y_pred > search_result['threshold']\ny_pred = y_pred.astype(int)\nprint(y_pred[:5])\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = df_final.qid\ndf_subm['prediction'] = y_pred\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}