{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"Using fastext model:\nhttps://github.com/keras-team/keras/blob/master/examples/imdb_fasttext.py\n\nInspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n\n(and other links in notebook)\n\nRemark:\nmodel overfits like hell..."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"print('loading word2vec model...')\nword2vec = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)\nprint('vocab:',len(word2vec.vocab))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77b7c5199c59943744495e62d7c0f73f68769e17"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf[\"question_text\"].fillna(\"_##_\",inplace=True)\nmax_len = df['question_text'].apply(lambda x:len(x)).max()\nprint('max length of sequences:',max_len)\n# df = df.sample(frac=0.1)\n\nprint('columns:',df.columns)\npd.set_option('display.max_columns',None)\nprint('df head:',df.head())\nprint('example of the question text values:',df['question_text'].head().values)\nprint('what values contains target:',df.target.unique())\n\nprint('Computing class weights....')\n#https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(df.target.values),\n                                                 df.target.values)\nprint('class_weights:',class_weights)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f41b23c1f3f4eed0d8d419974fe795b63f3df50b"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#dim of vectors\ndim = 300\n# max words in vocab\nnum_words = 50000\n# max number in questions\nmax_len = 200\n\nprint('Fiting tokenizer')\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=num_words)\ntokenizer.fit_on_texts(df['question_text'])\n\nprint('spliting data')\ndf_train,df_test = train_test_split(df)\n\nprint('text to sequence')\nx_train = tokenizer.texts_to_sequences(df_train['question_text'])\nx_test = tokenizer.texts_to_sequences(df_test['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx_train = pad_sequences(x_train,maxlen=max_len)\nx_test = pad_sequences(x_test, maxlen=max_len)\n\n## Get the target values\ny_train = df_train['target'].values\ny_test = df_test['target'].values\n\nprint(x_train.shape)\nprint(y_train.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db71c4ce3c1e3743f964c1a6e43a11644ee53cb4","scrolled":true},"cell_type":"code","source":"all_embs = word2vec.vectors\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(emb_mean,emb_std)\n\nprint(num_words,' from ',len(tokenizer.word_index.items()))\nnum_words = min(num_words, len(tokenizer.word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (num_words, dim))\n\n# embedding_matrix = np.zeros((num_words, dim))\ncount = 0\nfor word, i in tokenizer.word_index.items():\n    if i>=num_words:\n        break\n    if word in word2vec.vocab:\n        embedding_matrix[i] = word2vec.word_vec(word)\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix.shape)\nprint('Number of words not in vocab:',count)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55cb2ac1dca7de9fba51e8a7e5dba402159be302"},"cell_type":"code","source":"from keras.layers import Dense, Input,Embedding, Dropout, Activation, CuDNNLSTM,BatchNormalization\nfrom keras.layers import Bidirectional, GlobalMaxPool1D,GlobalAveragePooling1D\nfrom keras.models import Model\nfrom keras.callbacks import Callback,EarlyStopping\nfrom keras.engine import Layer\nfrom keras.initializers import Ones, Zeros\nimport keras.backend as K\nfrom keras import regularizers\nfrom keras import constraints\n\n# https://arxiv.org/abs/1607.06450\n# https://github.com/keras-team/keras/issues/3878\nclass LayerNormalization(Layer):\n    def __init__(self, **kwargs):\n        super(LayerNormalization, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        self.gain = self.add_weight(name='gain', shape=input_shape[-1:],\n                                    initializer=Ones(), trainable=True)\n        self.bias = self.add_weight(name='bias', shape=input_shape[-1:],\n                                    initializer=Zeros(), trainable=True)\n        super(LayerNormalization, self).build(input_shape)\n\n    def call(self, x, **kwargs):\n        mean = K.mean(x, axis=-1, keepdims=True)\n        std = K.std(x, axis=-1, keepdims=True)\n        # dot = *\n        # std+eps because of possible nans..\n        return self.gain * (x - mean) / (std + K.epsilon()) + self.bias\n\n    def compute_output_shape(self, input_shape):\n        return input_shape\n\ninp = Input(shape=(max_len,))\n#classic emb layer with pretrained weights\nx = Embedding(num_words, dim, weights=[embedding_matrix], trainable=True)(inp)\n# x = BatchNormalization()(x)\nx = GlobalAveragePooling1D()(x)\n# x = BatchNormalization()(x)\nx = Dense(1, activation=\"sigmoid\")(x)\n\nmodel = Model(inputs=inp, outputs=x)\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nprint(model.summary())\n\nhistory = model.fit(x_train,y_train, \n                      batch_size=512, \n                      validation_split=0.2,\n                      class_weight=class_weights,\n                      epochs=1000,                     \n                      callbacks=[EarlyStopping(patience=5)])\n\n_, ax = plt.subplots(1, 2, figsize=(12, 6))\nax[0].plot(history.history['loss'], label='loss')\nax[0].plot(history.history['val_loss'], label='val_loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['acc'], label='acc')\nax[1].plot(history.history['val_acc'], label='val_acc')\nax[1].legend()\nax[1].set_title('acc')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cde59508feb4a5c5f398128f9a5176e9a1912236"},"cell_type":"code","source":"#for train set\ny_pred = model.predict(x_train,batch_size=1024, verbose=1)\nsearch_result = threshold_search(y_train, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TRAINING SET:',classification_report(y_train,y_pred))\n\n\n#for test set\ny_pred = model.predict(x_test,batch_size=1024, verbose=1)\nsearch_result = threshold_search(y_test, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TEST SET:',classification_report(y_test,y_pred))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe1f23f3873f7dc4d67bbead51620009f759f0ee"},"cell_type":"markdown","source":"# Results from various runs\n\n## with non trainable embeddings\n\n    979591/979591 [==============================] - 5s 5us/step\n    {'threshold': 0.18, 'f1': 0.5533884446311493}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.97      0.96      0.97    918927\n              1       0.51      0.61      0.55     60664\n\n    avg / total       0.94      0.94      0.94    979591\n\n    326531/326531 [==============================] - 1s 4us/step\n    {'threshold': 0.18, 'f1': 0.5536153967787842}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.97      0.96      0.97    306385\n              1       0.51      0.61      0.55     20146\n\n    avg / total       0.95      0.94      0.94    326531\n\n\n\n## with trainable embeddings\n\n    979591/979591 [==============================] - 5s 5us/step\n    {'threshold': 0.27, 'f1': 0.6715829372296688}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    918927\n              1       0.63      0.72      0.67     60664\n\n    avg / total       0.96      0.96      0.96    979591\n\n    326531/326531 [==============================] - 1s 5us/step\n    {'threshold': 0.25, 'f1': 0.611690880567492}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306385\n              1       0.56      0.67      0.61     20146\n\n    avg / total       0.95      0.95      0.95    326531\n    \nOverfits...but still nice results.\n\n\n## with trainable embeddings and dim of emb vector = 200\n\n    979591/979591 [==============================] - 8s 8us/step\n    {'threshold': 0.21, 'f1': 0.6674156432805175}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    919157\n              1       0.63      0.71      0.67     60434\n\n    avg / total       0.96      0.96      0.96    979591\n\n    326531/326531 [==============================] - 3s 9us/step\n    {'threshold': 0.18, 'f1': 0.615255324324922}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.96      0.97    306155\n              1       0.56      0.68      0.62     20376\n\n    avg / total       0.95      0.95      0.95    326531\n\nStill overfits, but better results then before...which can be insignificat and from random variables. What starts to be clear si some hard cap with 67 score.\n\n## with trainable embeddings and dim of emb vector = 300\n\n    979591/979591 [==============================] - 12s 12us/step\n    {'threshold': 0.19, 'f1': 0.6684895085880024}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    918981\n              1       0.62      0.73      0.67     60610\n\n    avg / total       0.96      0.96      0.96    979591\n\n    326531/326531 [==============================] - 4s 12us/step\n    {'threshold': 0.17, 'f1': 0.6154084158415841}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.96      0.97    306331\n              1       0.56      0.69      0.62     20200\n\n    avg / total       0.95      0.95      0.95    326531\n\nOverfits and minor improvement in recall. \n\n## with trainable embeddings and dim of emb vector = 400\n\n    979591/979591 [==============================] - 16s 16us/step\n    {'threshold': 0.26, 'f1': 0.6631008624210988}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    919204\n              1       0.62      0.71      0.66     60387\n\n    avg / total       0.96      0.96      0.96    979591\n\n    326531/326531 [==============================] - 5s 16us/step\n    {'threshold': 0.23, 'f1': 0.618445328650499}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306108\n              1       0.57      0.68      0.62     20423\n\n    avg / total       0.95      0.95      0.95    326531\n    \n    \n## with trainable embeddings and dim of emb vector = 200 + layernorm\n\n    979591/979591 [==============================] - 29s 30us/step\n    {'threshold': 0.19, 'f1': 0.6803896944689282}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.98    918888\n              1       0.65      0.72      0.68     60703\n\n    avg / total       0.96      0.96      0.96    979591\n\n    326531/326531 [==============================] - 10s 30us/step\n    {'threshold': 0.16, 'f1': 0.5991003271537623}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306424\n              1       0.55      0.66      0.60     20107\n\n    avg / total       0.95      0.95      0.95    326531\n    \novertfit, quite bad actually\n\n## with trainable embeddings and dim of emb vector = 200 + batchnorm\n    979591/979591 [==============================] - 31s 32us/step\n    {'threshold': 0.19, 'f1': 0.6826963544106788}\n    RESULTS ON TRAINING SET:              precision    recall  f1-score   support\n\n              0       0.98      0.98      0.98    918888\n              1       0.66      0.71      0.68     60703\n\n    avg / total       0.96      0.96      0.96    979591\n\n    326531/326531 [==============================] - 10s 31us/step\n    {'threshold': 0.16, 'f1': 0.5965606943069591}\n    RESULTS ON TEST SET:              precision    recall  f1-score   support\n\n              0       0.98      0.97      0.97    306424\n              1       0.56      0.64      0.60     20107\n\n    avg / total       0.95      0.95      0.95    326531\n\n    overtfit, quite bad actually\n"},{"metadata":{"trusted":true,"_uuid":"3beb75a2101c603f182934220444bf573647220c"},"cell_type":"code","source":"#fit final model on all data\nprint('text to sequence')\nx = tokenizer.texts_to_sequences(df['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx = pad_sequences(x,maxlen=max_len)\n\n## Get the target values\ny = df['target'].values\n\nprint('fiting final model...')\nmodel.fit(x,y, batch_size=512, epochs=13,class_weight=class_weights)\n\ny_pred = model.predict(x,batch_size=1024, verbose=1)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint(classification_report(y,y_pred))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"#submission\nprint('Loading test data...')\ndf_final = pd.read_csv('../input/test.csv')\ndf_final[\"question_text\"].fillna(\"_##_\", inplace=True)\n\nx_final=tokenizer.texts_to_sequences(df_final['question_text'])\nx_final = pad_sequences(x_final,maxlen=max_len)\n\ny_pred = model.predict(x_final)\ny_pred = y_pred > search_result['threshold']\ny_pred = y_pred.astype(int)\nprint(y_pred[:5])\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = df_final.qid\ndf_subm['prediction'] = y_pred\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}