{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n\n(and other links in notebook)\n\nRemark:\nmodel overfits like hell..."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    print('\\nbest threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\nprint('loading word2vec model...')\nword2vec = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)\nprint('vocab:',len(word2vec.vocab))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77b7c5199c59943744495e62d7c0f73f68769e17"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf[\"question_text\"].fillna(\"_##_\",inplace=True)\nmax_len = df['question_text'].apply(lambda x:len(x)).max()\nprint('max length of sequences:',max_len)\n# df = df.sample(frac=0.1)\n\nprint('columns:',df.columns)\npd.set_option('display.max_columns',None)\nprint('df head:',df.head())\nprint('example of the question text values:',df['question_text'].head().values)\nprint('what values contains target:',df.target.unique())\n\nprint('Computing class weights....')\n#https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(df.target.values),\n                                                 df.target.values)\nprint('class_weights:',class_weights)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f41b23c1f3f4eed0d8d419974fe795b63f3df50b"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#dim of vectors\ndim = 300\n# max words in vocab\nnum_words = 50000\n# max number in questions\nmax_len = 100 \n\nprint('Fiting tokenizer')\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=num_words)\ntokenizer.fit_on_texts(df['question_text'])\n\nprint('spliting data')\ndf_train,df_test = train_test_split(df)\n\nprint('text to sequence')\nx_train = tokenizer.texts_to_sequences(df_train['question_text'])\nx_test = tokenizer.texts_to_sequences(df_test['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx_train = pad_sequences(x_train,maxlen=max_len)\nx_test = pad_sequences(x_test, maxlen=max_len)\n\n## Get the target values\ny_train = df_train['target'].values\ny_test = df_test['target'].values\n\nprint(x_train.shape)\nprint(y_train.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db71c4ce3c1e3743f964c1a6e43a11644ee53cb4","scrolled":true},"cell_type":"code","source":"all_embs = word2vec.vectors\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(emb_mean,emb_std)\n\nprint(num_words,' from ',len(tokenizer.word_index.items()))\nnum_words = min(num_words, len(tokenizer.word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (num_words, dim))\n\n# embedding_matrix = np.zeros((num_words, dim))\ncount = 0\nfor word, i in tokenizer.word_index.items():\n    if i>=num_words:\n        break\n    if word in word2vec.vocab:\n        embedding_matrix[i] = word2vec.word_vec(word)\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix.shape)\nprint('Number of words not in vocab:',count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55cb2ac1dca7de9fba51e8a7e5dba402159be302"},"cell_type":"code","source":"from keras.layers import Dense, Input,Embedding, Dropout, Activation, CuDNNLSTM,BatchNormalization\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras.callbacks import Callback,EarlyStopping\nfrom keras.engine import Layer\nfrom keras.initializers import Ones, Zeros\nimport keras.backend as K\nfrom keras import regularizers\nfrom keras import constraints\n\n# https://arxiv.org/abs/1607.06450\n# https://github.com/keras-team/keras/issues/3878\nclass LayerNormalization(Layer):\n    def __init__(self, **kwargs):\n        super(LayerNormalization, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.gain = self.add_weight(name='gain', shape=input_shape[-1:],\n                                    initializer=Ones(), trainable=True)\n        self.bias = self.add_weight(name='bias', shape=input_shape[-1:],\n                                    initializer=Zeros(), trainable=True)\n        super(LayerNormalization, self).build(input_shape)\n\n    def call(self, x, **kwargs):\n        mean = K.mean(x, axis=-1, keepdims=True)\n        std = K.std(x, axis=-1, keepdims=True)\n        # dot = *\n        # std+eps because of possible nans..\n        return self.gain * (x - mean) / (std + K.epsilon()) + self.bias\n\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\n#model looks to be from here: https://www.kaggle.com/CVxTz/keras-bidirectional-lstm-baseline-lb-0-069\n\n    \ninp = Input(shape=(max_len,))\n#classic emb layer with pretrained weights\nx = Embedding(num_words, dim, weights=[embedding_matrix], trainable=False)(inp)\n#seq2seq?\n# x = LayerNormalization()(x)\nx = Bidirectional(CuDNNLSTM(64, \n                            return_sequences=True, \n#                             kernel_regularizer = regularizers.l2(0.001),\n#                             recurrent_regularizer = regularizers.l2(0.001),\n#                             kernel_constraint=constraints.MaxNorm(axis=-1), \n#                             recurrent_constraint=constraints.MaxNorm(axis=-1)\n                           ))(x)\n# limitations of CuDNN https://www.reddit.com/r/MLQuestions/comments/9an2y0/keras_cudnnlstm_is_it_worth_the_drawbacks/\n# so looks like by this that layer normalization can be used: http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n# but batch normalization can't\n# x = LayerNormalization()(x)\n#why is the max here?\nx = GlobalMaxPool1D()(x)\n#why is the dense here?\nx = Dense(16, activation=\"relu\")(x)\n# x = LayerNormalization()(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\n\nmodel = Model(inputs=inp, outputs=x)\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nprint(model.summary())\n\n# for commiting the model to competition i need to comment these sections....otherwise the running time will be more then 2h on gpu...\nhistory = model.fit(x_train,y_train, \n                      batch_size=512, \n                      validation_split=0.2,\n                      class_weight=class_weights,\n                      epochs=100,\n                      #overfits rather soon\n                      callbacks=[EarlyStopping(patience=2)])\n\n_, ax = plt.subplots(1, 2, figsize=(12, 6))\nax[0].plot(history.history['loss'], label='loss')\nax[0].plot(history.history['val_loss'], label='val_loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['acc'], label='acc')\nax[1].plot(history.history['val_acc'], label='val_acc')\nax[1].legend()\nax[1].set_title('acc')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cde59508feb4a5c5f398128f9a5176e9a1912236"},"cell_type":"code","source":"#for train set\ny_pred = model.predict(x_train,batch_size=1024, verbose=1)\nsearch_result = threshold_search(y_train, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TRAINING SET:\\n',classification_report(y_train,y_pred))\n\n\n#for test set\ny_pred = model.predict(x_test,batch_size=1024, verbose=1)\nsearch_result = threshold_search(y_test, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TEST SET:\\n',classification_report(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"82016ead496e27b0020905dc08753a9b62093fc7"},"cell_type":"markdown","source":"# Results\n\n## without regul , non trainable emb\n    979591/979591 [==============================] - 50s 51us/step\n    threshold = 0.990000 | score = 0.000099\n    best threshold is  0.330000 with score 0.725006\n    {'threshold': 0.33, 'f1': 0.7250061303106289}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.98      0.98      0.98    918850\n              1       0.70      0.75      0.73     60741\n\n    avg / total       0.97      0.96      0.97    979591\n\n    326531/326531 [==============================] - 16s 50us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.280000 with score 0.655881\n    {'threshold': 0.28, 'f1': 0.6558810668998688}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    306462\n              1       0.61      0.71      0.66     20069\n\n    avg / total       0.96      0.95      0.96    326531\n\n## without regul , non trainable emb, with Dropout 0.1\n    979591/979591 [==============================] - 49s 50us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.460000 with score 0.727626\n    {'threshold': 0.46, 'f1': 0.7276255725700638}\n    RESULTS ON TRAINING SET:\n                  precision    recall  f1-score   support\n\n              0       0.98      0.98      0.98    918850\n              1       0.70      0.76      0.73     60741\n\n    avg / total       0.97      0.96      0.97    979591\n\n    326531/326531 [==============================] - 16s 50us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.420000 with score 0.656067\n    {'threshold': 0.42, 'f1': 0.65606690715254}\n    RESULTS ON TEST SET:\n                  precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    306462\n              1       0.61      0.71      0.66     20069\n\n    avg / total       0.96      0.95      0.96    326531\n\n\n## without regul , trainable emb, with Dropout 0.1\n\n    979591/979591 [==============================] - 49s 50us/step\n    threshold = 0.990000 | score = 0.031973\n    best threshold is  0.460000 with score 0.826382\n    {'threshold': 0.46, 'f1': 0.8263816852821267}\n    RESULTS ON TRAINING SET:\n                  precision    recall  f1-score   support\n\n              0       0.99      0.99      0.99    918850\n              1       0.81      0.85      0.83     60741\n\n    avg / total       0.98      0.98      0.98    979591\n\n    326531/326531 [==============================] - 16s 50us/step\n    threshold = 0.990000 | score = 0.016393\n    best threshold is  0.400000 with score 0.629146\n    {'threshold': 0.4, 'f1': 0.629146426092991}\n    RESULTS ON TEST SET:\n                  precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306462\n              1       0.59      0.68      0.63     20069\n\n    avg / total       0.95      0.95      0.95    326531\n\n\n## without regul , trainable emb\n\n    979591/979591 [==============================] - 50s 51us/step\n    threshold = 0.990000 | score = 0.009536\n    best threshold is  0.390000 with score 0.840622\n    {'threshold': 0.39, 'f1': 0.8406221710147314}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.99      0.99      0.99    918850\n              1       0.84      0.84      0.84     60741\n\n    avg / total       0.98      0.98      0.98    979591\n\n    326531/326531 [==============================] - 16s 50us/step\n    threshold = 0.990000 | score = 0.004374\n    best threshold is  0.250000 with score 0.631671\n    {'threshold': 0.25, 'f1': 0.6316712025462428}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306462\n              1       0.59      0.68      0.63     20069\n\n    avg / total       0.95      0.95      0.95    326531\n\n## with regularizers, non trainable emb\n    \n    kernel_regularizer = regularizers.l2(0.001),\n    recurrent_regularizer = regularizers.l2(0.001)\n    \n    best threshold is  0.250000 with score 0.635888\n             precision    recall  f1-score   support\n\n          0       0.98      0.97      0.97    306130\n          1       0.59      0.69      0.64     20401\n\n    avg / total       0.95      0.95      0.95    326531\n\n## with layernorm, without regularizers, non trainable emb\n\n    326531/326531 [==============================] - 21s 65us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.240000 with score 0.652420\n                 precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306130\n              1       0.60      0.71      0.65     20401\n\n    avg / total       0.96      0.95      0.95    326531\n    \nBut graphs show overfit.\n    \n\n## with layernorm, with regularizers, non trainable emb\n\n       979591/979591 [==============================] - 64s 65us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.260000 with score 0.650079\n    {'threshold': 0.26, 'f1': 0.6500788180385126}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    919182\n              1       0.61      0.70      0.65     60409\n\n    avg / total       0.96      0.95      0.96    979591\n\n    326531/326531 [==============================] - 21s 64us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.260000 with score 0.642098\n    {'threshold': 0.26, 'f1': 0.6420979986197378}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306130\n              1       0.60      0.68      0.64     20401\n\n    avg / total       0.96      0.95      0.95    326531\n    \n    Doesn't overfit.\n    \n## with layernorm, with regularizers,  trainable emb\n\n    979591/979591 [==============================] - 63s 65us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.420000 with score 0.734747\n    {'threshold': 0.42, 'f1': 0.7347472422512937}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.99      0.98      0.98    919182\n              1       0.70      0.78      0.73     60409\n\n    avg / total       0.97      0.97      0.97    979591\n\n    326531/326531 [==============================] - 21s 65us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.420000 with score 0.650728\n    {'threshold': 0.42, 'f1': 0.6507277969200478}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    306130\n              1       0.62      0.68      0.65     20401\n\n    avg / total       0.96      0.95      0.96    326531\n    \nOverfits immediatly....in 2 epochs.\n\n## with layernorm, with regularizers,  trainable emb and constraints MaxNorm\n\n    979591/979591 [==============================] - 64s 65us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.360000 with score 0.760183\n    {'threshold': 0.36, 'f1': 0.7601830919533802}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.99      0.98      0.98    919182\n              1       0.73      0.79      0.76     60409\n\n    avg / total       0.97      0.97      0.97    979591\n\n    326531/326531 [==============================] - 21s 65us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.290000 with score 0.643731\n    {'threshold': 0.29, 'f1': 0.6437309901993917}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306130\n              1       0.60      0.70      0.64     20401\n\n    avg / total       0.96      0.95      0.95    326531\n    \nOverfits. On test not an improvement....\n\n## with layernorm, without regularizers, trainable emb\n\n    979591/979591 [==============================] - 64s 65us/step\n    threshold = 0.990000 | score = 0.014331\n    best threshold is  0.430000 with score 0.823383\n    {'threshold': 0.43, 'f1': 0.8233833284318531}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.99      0.99      0.99    919182\n              1       0.81      0.83      0.82     60409\n\n    avg / total       0.98      0.98      0.98    979591\n\n    326531/326531 [==============================] - 21s 65us/step\n    threshold = 0.990000 | score = 0.005083\n    best threshold is  0.340000 with score 0.649846\n    {'threshold': 0.34, 'f1': 0.6498460683780468}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    306130\n              1       0.62      0.69      0.65     20401\n\n    avg / total       0.96      0.95      0.95    326531\n    \n    Crazy results on train.\n    \n## with layernorm, without regularizers, trainable emb, maxlen = 200\n\n    979591/979591 [==============================] - 125s 128us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.580000 with score 0.820049\n    {'threshold': 0.58, 'f1': 0.8200489795918368}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.99      0.99      0.99    918924\n              1       0.81      0.83      0.82     60667\n\n    avg / total       0.98      0.98      0.98    979591\n\n    326531/326531 [==============================] - 41s 127us/step\n    threshold = 0.990000 | score = 0.000000\n    best threshold is  0.490000 with score 0.644622\n    {'threshold': 0.49, 'f1': 0.6446220093490224}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306388\n              1       0.60      0.69      0.64     20143\n\n    avg / total       0.96      0.95      0.95    326531\n    \n## with layernorm, without regularizers, trainable emb, maxlen = 300"},{"metadata":{"trusted":true,"_uuid":"3beb75a2101c603f182934220444bf573647220c"},"cell_type":"code","source":"#fit final model on all data\nprint('text to sequence')\nx = tokenizer.texts_to_sequences(df['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx = pad_sequences(x,maxlen=max_len)\n\n## Get the target values\ny = df['target'].values\n\nprint('fiting final model...')\nhistory = model.fit(x,y, batch_size=512, epochs=6,class_weight=class_weights)\n\n_, ax = plt.subplots(1, 2, figsize=(12, 6))\nax[0].plot(history.history['loss'], label='loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['acc'], label='acc')\nax[1].legend()\nax[1].set_title('acc')\n\nplt.show()\n\ny_pred = model.predict(x,batch_size=1024, verbose=1)\nsearch_result = threshold_search(y, y_pred)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint(classification_report(y,y_pred))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"#submission\nprint('Loading test data...')\ndf_final = pd.read_csv('../input/test.csv')\ndf_final[\"question_text\"].fillna(\"_##_\", inplace=True)\n\nx_final=tokenizer.texts_to_sequences(df_final['question_text'])\nx_final = pad_sequences(x_final,maxlen=max_len)\n\ny_pred = model.predict(x_final)\ny_pred = y_pred > search_result['threshold']\ny_pred = y_pred.astype(int)\nprint(y_pred[:5])\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = df_final.qid\ndf_subm['prediction'] = y_pred\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}