{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"### Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n* https://ai.google/research/pubs/pub46697\n* https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n\n\n(and other links in notebook)\n\nRemark:\nmodel overfits like hell...\n\nv6.1:\nincreased size of conv from 32 -> 100\nFor commit I have to disable training and tuning stage and fit on whole model, otherwise the running time is longer than 2 hours.\n\nv10:\n- use only one model\n- add lstm to cnn\n\nv11:\ntime distributed\n\nv12:\nmodelcheck+lr on plateu \n\nv13:\nloading best performing\n\nv14:\nresidual\n\nremoving word2vec\n\n----\n\nadding clr\n\n----\n\nadding unfreeze\n    "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n\nSEED = 2018\n\nnp.random.seed(SEED)\ntf.set_random_seed(SEED)\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    print('\\nbest threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77b7c5199c59943744495e62d7c0f73f68769e17"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf[\"question_text\"].fillna(\"_##_\",inplace=True)\nmax_len = df['question_text'].apply(lambda x:len(x)).max()\nprint('max length of sequences:',max_len)\n# df = df.sample(frac=0.1)\n\nprint('columns:',df.columns)\npd.set_option('display.max_columns',None)\nprint('df head:',df.head())\nprint('example of the question text values:',df['question_text'].head().values)\nprint('what values contains target:',df.target.unique())\n\nprint('Computing class weights....')\n#https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(df.target.values),\n                                                 df.target.values)\nprint('class_weights:',class_weights)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f41b23c1f3f4eed0d8d419974fe795b63f3df50b"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#dim of vectors\ndim = 300\n# max words in vocab\nnum_words = 50000\n# max number in questions\nmax_len = 100 \n\nprint('Fiting tokenizer')\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=num_words)\ntokenizer.fit_on_texts(df['question_text'])\n\nprint('spliting data')\ndf_train,df_test = train_test_split(df, random_state=1)\n\nprint('text to sequence')\nx_train = tokenizer.texts_to_sequences(df_train['question_text'])\nx_test = tokenizer.texts_to_sequences(df_test['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx_train = pad_sequences(x_train,maxlen=max_len)\nx_test = pad_sequences(x_test, maxlen=max_len)\n\n## Get the target values\ny_train = df_train['target'].values\ny_test = df_test['target'].values\n\nprint(x_train.shape)\nprint(y_train.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f1ed31984c07cbb1a95c250e0dadf9eb649e5a3"},"cell_type":"code","source":"print('Glove ... ')\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt'))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n\nword_index = tokenizer.word_index\n# num_words = min(num_words, len(word_index))\nembedding_matrix_glov = np.random.normal(emb_mean, emb_std, (num_words, dim))\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_glov[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9712758dc3cab221d6e57f43e5eb00224386d7d5"},"cell_type":"code","source":"print('Para...')\nEMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n\nword_index = tokenizer.word_index\n# num_words = min(num_words, len(word_index))\nembedding_matrix_para = np.random.normal(emb_mean, emb_std, (num_words, dim))\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_para[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d17e4323bef806ca6fa47c1edf8a3764a082caa3"},"cell_type":"code","source":"matrixes = [embedding_matrix_glov,embedding_matrix_para]\n\nmatrix = np.mean(matrixes,axis=0)\n\ndel embedding_matrix_glov,embedding_matrix_para\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55cb2ac1dca7de9fba51e8a7e5dba402159be302","scrolled":true},"cell_type":"code","source":"from keras.layers import Dense, Input,Embedding, Dropout, Activation, CuDNNLSTM,BatchNormalization,concatenate,SpatialDropout1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Concatenate, GlobalAveragePooling1D,Average,Conv1D,GlobalMaxPooling1D,AlphaDropout\nfrom keras.layers import MaxPooling1D,UpSampling1D,RepeatVector,LSTM,TimeDistributed,Flatten, CuDNNGRU, Add\nfrom keras.models import Model\nfrom keras.callbacks import Callback,EarlyStopping,ModelCheckpoint, ReduceLROnPlateau\nfrom keras.engine import Layer\nfrom keras.initializers import Ones, Zeros\nimport keras.backend as K\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras import optimizers\nfrom keras import initializers\nimport warnings\n\n# https://www.kaggle.com/suicaokhoailang/lstm-attention-baseline-0-652-lb\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim\n\ndef f1(y_true, y_pred):\n    '''\n    metric from here \n    https://stackoverflow.com/questions/43547402/how-to-calculate-f1-macro-in-keras\n    '''\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\n\nclass CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())\n\n\nclass Metrics(Callback):\n    def on_train_begin(self, logs={}):\n        self.val_f1s = []\n    \n    def _threshold_search(self,y_true, y_proba):\n        best_threshold = 0\n        best_score = 0\n        for threshold in [i * 0.01 for i in range(100)]:\n            score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n            if score > best_score:\n                best_threshold = threshold\n                best_score = score\n        search_result = {'threshold': best_threshold, 'f1': best_score}\n        return search_result\n    \n    \n    def on_epoch_end(self, epoch, logs={}):\n        y_val = self.validation_data[1]\n        y_pred = self.model.predict(self.validation_data[0])\n\n        with warnings.catch_warnings():\n            warnings.simplefilter(\"ignore\")\n            search_result = self._threshold_search(y_val, y_pred)\n        \n        _val_f1 = search_result['f1']\n        self.val_f1s.append(_val_f1)\n        print ('— val_f1_on_epoch_end: %f '%(_val_f1))\n        return\n\n# https://ai.google/research/pubs/pub46697\nadam = optimizers.Adam()\nprint('LR:',K.eval(adam.lr))\n# 0.001 = learning rate in adam\n# optimal batch size ~ eps *N, where eps = learning rate and N = training size\nbatch_size = int(x_train.shape[0]*K.eval(adam.lr))\nprint('Batch size = ',batch_size)\n\n\n# Yoon Kim's shallow cnn model: https://arxiv.org/pdf/1408.5882.pdf\n# https://github.com/raghakot/keras-text/blob/master/keras_text/models/sequence_encoders.py\n#overfitting:\n# https://stackoverflow.com/questions/43156397/how-to-avoid-overfitting-in-the-given-convnet\n# http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/\n# https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n#word2vec    \ndef get_model(opt, trainable=False):\n    inp1 = Input(shape=(max_len,))\n    x = Embedding(num_words, dim, weights=[matrix],trainable = trainable,)(inp1)\n    x = SpatialDropout1D(0.1)(x)\n    x1 = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x)\n    x = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x1)\n    x = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x)\n    x = Bidirectional(CuDNNLSTM(int(max_len/2), return_sequences=True))(x)\n\n    x = Add()([x1,x])\n    atten_1 = Attention(max_len)(x) \n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n\n    x = concatenate([atten_1, avg_pool, max_pool])\n\n    #classification dense net\n    x = Dense(100, activation=\"relu\")(x)\n    x = Dense(1, activation='sigmoid')(x)\n\n\n    model = Model(inputs=inp1, outputs=x)\n\n    model.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])\n\n    print(model.summary())\n    \n    return model\n\n# for commiting the model to competition i need to comment these sections....otherwise the running time will be more then 2h on gpu...\npatience = 2\nmodelcheck = ModelCheckpoint('model.h5',save_best_only=True)\nstop = EarlyStopping(patience=patience)\nreduce = CyclicLR(base_lr=0.001, \n                      max_lr=0.004,\n                      step_size=300., \n                      mode='exp_range',\n                       gamma=0.99994)\nmetrics = Metrics()\n\nmodel = get_model(adam)\n\nhistory = model.fit(x_train,y_train, \n                      batch_size=batch_size, \n                      validation_split=0.2,\n                      epochs=4,\n                      #overfits rather soon\n                      callbacks=[modelcheck,stop,reduce, metrics])\n\n#for train set\ny_pred = model.predict(x_train,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_train, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TRAINING SET:\\n',classification_report(y_train,y_pred))\n\n\n#for test set\ny_pred = model.predict(x_test,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_test, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TEST SET:\\n',classification_report(y_test,y_pred))\n\nprint('Pretraining finished...Unfreezing...')\n\nadam = optimizers.Adam()\n#load best performing\nmodel = get_model(adam,True)\nmodel.load_weights('model.h5')\n\nmodelcheck = ModelCheckpoint('model.h5',save_best_only=True)\nstop = EarlyStopping(patience=patience)\nreduce = CyclicLR(base_lr=0.001, \n                      max_lr=0.004,\n                      step_size=300., \n                      mode='exp_range',\n                       gamma=0.99994)\n\nhistory = model.fit(x_train,y_train, \n                      batch_size=batch_size, \n                      validation_split=0.2,\n                      epochs=4,\n                      #overfits rather soon\n                      callbacks=[modelcheck,stop,reduce, metrics])\n\n\nprint('training finished...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cde59508feb4a5c5f398128f9a5176e9a1912236","scrolled":true},"cell_type":"code","source":"#load best performing\n\nmodel.load_weights('model.h5')\n\n#for train set\ny_pred = model.predict(x_train,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_train, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TRAINING SET:\\n',classification_report(y_train,y_pred))\n\n\n#for test set\ny_pred = model.predict(x_test,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_test, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TEST SET:\\n',classification_report(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"#submission\nprint('Loading test data...')\ndf_final = pd.read_csv('../input/test.csv')\ndf_final[\"question_text\"].fillna(\"_##_\", inplace=True)\n\nx_final=tokenizer.texts_to_sequences(df_final['question_text'])\nx_final = pad_sequences(x_final,maxlen=max_len)\n\ny_pred = model.predict(x_final,batch_size=batch_size,verbose=1)\ny_pred = y_pred > search_result['threshold']\ny_pred = y_pred.astype(int)\nprint(y_pred[:5])\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = df_final.qid\ndf_subm['prediction'] = y_pred\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}