{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n* https://ai.google/research/pubs/pub46697\n* https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n\n\n(and other links in notebook)\n\nRemark:\nmodel overfits like hell...\n\nv6.1:\nincreased size of conv from 32 -> 100\nFor commit I have to disable training and tuning stage and fit on whole model, otherwise the running time is longer than 2 hours.\n\nv10:\n- use only one model\n- add lstm to cnn\n\nv11:\nnormalization\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    print('\\nbest threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77b7c5199c59943744495e62d7c0f73f68769e17"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf[\"question_text\"].fillna(\"_##_\",inplace=True)\nmax_len = df['question_text'].apply(lambda x:len(x)).max()\nprint('max length of sequences:',max_len)\n# df = df.sample(frac=0.1)\n\nprint('columns:',df.columns)\npd.set_option('display.max_columns',None)\nprint('df head:',df.head())\nprint('example of the question text values:',df['question_text'].head().values)\nprint('what values contains target:',df.target.unique())\n\nprint('Computing class weights....')\n#https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(df.target.values),\n                                                 df.target.values)\nprint('class_weights:',class_weights)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f41b23c1f3f4eed0d8d419974fe795b63f3df50b"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#dim of vectors\ndim = 300\n# max words in vocab\nnum_words = 50000\n# max number in questions\nmax_len = 100 \n\nprint('Fiting tokenizer')\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=num_words)\ntokenizer.fit_on_texts(df['question_text'])\n\nprint('spliting data')\ndf_train,df_test = train_test_split(df, random_state=1)\n\nprint('text to sequence')\nx_train = tokenizer.texts_to_sequences(df_train['question_text'])\nx_test = tokenizer.texts_to_sequences(df_test['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx_train = pad_sequences(x_train,maxlen=max_len)\nx_test = pad_sequences(x_test, maxlen=max_len)\n\n## Get the target values\ny_train = df_train['target'].values\ny_test = df_test['target'].values\n\nprint(x_train.shape)\nprint(y_train.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db71c4ce3c1e3743f964c1a6e43a11644ee53cb4","scrolled":true},"cell_type":"code","source":"# https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\nprint('loading word2vec model...')\nword2vec = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)\nprint('vocab:',len(word2vec.vocab))\n\nall_embs = word2vec.vectors\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(emb_mean,emb_std)\n\nprint(num_words,' from ',len(tokenizer.word_index.items()))\n# num_words = min(num_words, len(tokenizer.word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (num_words, dim))\n\n# embedding_matrix = np.zeros((num_words, dim))\ncount = 0\nfor word, i in tokenizer.word_index.items():\n    if i>=num_words:\n        break\n    if word in word2vec.vocab:\n        embedding_matrix[i] = word2vec.word_vec(word)\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix.shape)\nprint('Number of words not in vocab:',count)\n\ndel word2vec\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4aea160023e2512b71c3ae6f6ce8b6099d1aed16"},"cell_type":"markdown","source":"# SpatialDropout1D\n\nsrc: https://keras.io/layers/core/\n\n    This version performs the same function as Dropout, however it drops entire 1D feature maps instead of individual elements. If adjacent frames within feature maps are strongly correlated (as is normally the case in early convolution layers) then regular dropout will not regularize the activations and will otherwise just result in an effective learning rate decrease. In this case, SpatialDropout1D will help promote independence between feature maps and should be used instead.\n\n# selu\n\nsrc: https://keras.io/activations/\n\nSELU is equal to: scale * elu(x, alpha), where alpha and scale are pre-defined constants. The values of alpha and scale are chosen so that the mean and variance of the inputs are preserved between two consecutive layers as long as the weights are initialized correctly (see lecun_normal initialization) and the number of inputs is \"large enough\" (see references for more information)."},{"metadata":{"trusted":true,"_uuid":"55cb2ac1dca7de9fba51e8a7e5dba402159be302","scrolled":true},"cell_type":"code","source":"from keras.layers import Dense, Input,Embedding, Dropout, Activation, CuDNNLSTM,BatchNormalization,concatenate,SpatialDropout1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Concatenate, GlobalAveragePooling1D,Average,Conv1D,GlobalMaxPooling1D,AlphaDropout\nfrom keras.models import Model\nfrom keras.callbacks import Callback,EarlyStopping\nfrom keras.engine import Layer\nfrom keras.initializers import Ones, Zeros\nimport keras.backend as K\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras import optimizers\n\n# https://ai.google/research/pubs/pub46697\nadam = optimizers.Adam()\nprint('LR:',K.eval(adam.lr))\n# 0.001 = learning rate in adam\n# optimal batch size ~ eps *N, where eps = learning rate and N = training size\nbatch_size = int(x_train.shape[0]*K.eval(adam.lr))\nprint('Batch size = ',batch_size)\n\n\n# Yoon Kim's shallow cnn model: https://arxiv.org/pdf/1408.5882.pdf\n# https://github.com/raghakot/keras-text/blob/master/keras_text/models/sequence_encoders.py\n#overfitting:\n# https://stackoverflow.com/questions/43156397/how-to-avoid-overfitting-in-the-given-convnet\n# http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/\n# https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n#word2vec    \ninp1 = Input(shape=(max_len,))\n# emb layer adds the most params to net...it's responsible for overfit? let's try setting them to non-trainable...\n# so it can be trainable, but needs to be regularized, otherwise it contributes to overfiting\ne1 = Embedding(num_words, dim, weights=[embedding_matrix],trainable = True,embeddings_regularizer=regularizers.l2(0.0001))(inp1)\n# e1 = Bidirectional(CuDNNLSTM(50,return_sequences=True))(e1)\ne1 = SpatialDropout1D(0.1)(e1)\npooled_tensors = []\nkernel_sizes = [3, 4, 5]\nfor kernel_size in kernel_sizes:    \n    l1 = Conv1D(64, kernel_size=kernel_size, activation='selu',kernel_regularizer=regularizers.l2(0.0001))(e1)\n    l1 = GlobalMaxPooling1D()(l1)\n    pooled_tensors.append(l1)\n\nif len(kernel_sizes) > 1:\n    x = concatenate(pooled_tensors, axis=-1)\nelse:\n    x = pooled_tensors[0]\n\nx = BatchNormalization()(x)\nx = AlphaDropout(0.1)(x)\nx = Dense(1, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001))(x)\n\nmodel = Model(inputs=inp1, outputs=x)\n\nmodel.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy'])\n\nprint(model.summary())\n\n# for commiting the model to competition i need to comment these sections....otherwise the running time will be more then 2h on gpu...\nhistory = model.fit(x_train,y_train, \n                      batch_size=batch_size, \n                      validation_split=0.33,\n                      epochs=100,\n                      #overfits rather soon\n                      callbacks=[EarlyStopping(patience=2)])\n\nprint('training done....')\nprint('LR:',K.eval(adam.lr))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da921fc2d216b1e2db6819ea13ab0b0c4a0d731d"},"cell_type":"code","source":"_, ax = plt.subplots(1, 2, figsize=(12, 6))\nax[0].plot(history.history['loss'], label='loss')\nax[0].plot(history.history['val_loss'], label='val_loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['acc'], label='acc')\nax[1].plot(history.history['val_acc'], label='val_acc')\nax[1].legend()\nax[1].set_title('acc')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cde59508feb4a5c5f398128f9a5176e9a1912236","scrolled":true},"cell_type":"code","source":"#for train set\ny_pred = model.predict(x_train,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_train, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TRAINING SET:\\n',classification_report(y_train,y_pred))\n\n\n#for test set\ny_pred = model.predict(x_test,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_test, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TEST SET:\\n',classification_report(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3beb75a2101c603f182934220444bf573647220c"},"cell_type":"code","source":"#fit final model on all data\nprint('text to sequence')\nx = tokenizer.texts_to_sequences(df['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx = pad_sequences(x,maxlen=max_len)\n\n## Get the target values\ny = df['target'].values\n\nprint('fiting final model...')\n# sadly overfits, so 2 epochs top\nhistory = model.fit(x,y, batch_size=batch_size, epochs=9)\n\nprint('fitting on full data done...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49e1bd5b69b428e57ad8248868d4b23c9c6f9d84"},"cell_type":"code","source":"_, ax = plt.subplots(1, 2, figsize=(12, 6))\nax[0].plot(history.history['loss'], label='loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['acc'], label='acc')\nax[1].legend()\nax[1].set_title('acc')\n\nplt.show()\n\ny_pred = model.predict(x,batch_size=batch_size, verbose=1)\n# search_result = threshold_search(y, y_pred)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint(classification_report(y,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"#submission\nprint('Loading test data...')\ndf_final = pd.read_csv('../input/test.csv')\ndf_final[\"question_text\"].fillna(\"_##_\", inplace=True)\n\nx_final=tokenizer.texts_to_sequences(df_final['question_text'])\nx_final = pad_sequences(x_final,maxlen=max_len)\n\ny_pred = model.predict(x_final,batch_size=batch_size,verbose=1)\ny_pred = y_pred > search_result['threshold']\ny_pred = y_pred.astype(int)\nprint(y_pred[:5])\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = df_final.qid\ndf_subm['prediction'] = y_pred\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}