{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"### Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n* https://ai.google/research/pubs/pub46697\n* https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n\n\n(and other links in notebook)\n\nRemark:\nmodel overfits like hell...\n\nv6.1:\nincreased size of conv from 32 -> 100\nFor commit I have to disable training and tuning stage and fit on whole model, otherwise the running time is longer than 2 hours.\n\nv10:\n- use only one model\n- add lstm to cnn\n\nv11:\ntime distributed"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n\nSEED = 2018\n\nnp.random.seed(SEED)\ntf.set_random_seed(SEED)\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    print('\\nbest threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77b7c5199c59943744495e62d7c0f73f68769e17"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf[\"question_text\"].fillna(\"_##_\",inplace=True)\nmax_len = df['question_text'].apply(lambda x:len(x)).max()\nprint('max length of sequences:',max_len)\n# df = df.sample(frac=0.1)\n\nprint('columns:',df.columns)\npd.set_option('display.max_columns',None)\nprint('df head:',df.head())\nprint('example of the question text values:',df['question_text'].head().values)\nprint('what values contains target:',df.target.unique())\n\nprint('Computing class weights....')\n#https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(df.target.values),\n                                                 df.target.values)\nprint('class_weights:',class_weights)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f41b23c1f3f4eed0d8d419974fe795b63f3df50b"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#dim of vectors\ndim = 300\n# max words in vocab\nnum_words = 50000\n# max number in questions\nmax_len = 100 \n\nprint('Fiting tokenizer')\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=num_words)\ntokenizer.fit_on_texts(df['question_text'])\n\nprint('spliting data')\ndf_train,df_test = train_test_split(df, random_state=1)\n\nprint('text to sequence')\nx_train = tokenizer.texts_to_sequences(df_train['question_text'])\nx_test = tokenizer.texts_to_sequences(df_test['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx_train = pad_sequences(x_train,maxlen=max_len)\nx_test = pad_sequences(x_test, maxlen=max_len)\n\n## Get the target values\ny_train = df_train['target'].values\ny_test = df_test['target'].values\n\nprint(x_train.shape)\nprint(y_train.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db71c4ce3c1e3743f964c1a6e43a11644ee53cb4","scrolled":true},"cell_type":"code","source":"# https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\nprint('loading word2vec model...')\nword2vec = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)\nprint('vocab:',len(word2vec.vocab))\n\nall_embs = word2vec.vectors\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(emb_mean,emb_std)\n\nprint(num_words,' from ',len(tokenizer.word_index.items()))\n# num_words = min(num_words, len(tokenizer.word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (num_words, dim))\n\n# embedding_matrix = np.zeros((num_words, dim))\ncount = 0\nfor word, i in tokenizer.word_index.items():\n    if i>=num_words:\n        break\n    if word in word2vec.vocab:\n        embedding_matrix[i] = word2vec.word_vec(word)\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix.shape)\nprint('Number of words not in vocab:',count)\n\ndel word2vec\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f1ed31984c07cbb1a95c250e0dadf9eb649e5a3"},"cell_type":"code","source":"print('Glove ... ')\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt'))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n\nword_index = tokenizer.word_index\n# num_words = min(num_words, len(word_index))\nembedding_matrix_glov = np.random.normal(emb_mean, emb_std, (num_words, dim))\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_glov[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9712758dc3cab221d6e57f43e5eb00224386d7d5"},"cell_type":"code","source":"print('Para...')\nEMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n\nword_index = tokenizer.word_index\n# num_words = min(num_words, len(word_index))\nembedding_matrix_para = np.random.normal(emb_mean, emb_std, (num_words, dim))\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_para[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74e18c69b73b22cf6f8dbf7a1ba4b4bf4e1042b7"},"cell_type":"code","source":"print('Wiki...')\nEMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\n\nprint(len(all_embs))\n\nword_index = tokenizer.word_index\nembedding_matrix_wiki = np.random.normal(emb_mean, emb_std, (num_words, dim))\n\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_wiki[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_wiki.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55cb2ac1dca7de9fba51e8a7e5dba402159be302","scrolled":true},"cell_type":"code","source":"from keras.layers import Dense, Input,Embedding, Dropout, Activation, CuDNNLSTM,BatchNormalization,concatenate,SpatialDropout1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Concatenate, GlobalAveragePooling1D,Average,Conv1D,GlobalMaxPooling1D,AlphaDropout\nfrom keras.layers import MaxPooling1D,UpSampling1D,RepeatVector,LSTM,TimeDistributed,Flatten\nfrom keras.models import Model\nfrom keras.callbacks import Callback,EarlyStopping\nfrom keras.engine import Layer\nfrom keras.initializers import Ones, Zeros\nimport keras.backend as K\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras import optimizers\n\ndef f1(y_true, y_pred):\n    '''\n    metric from here \n    https://stackoverflow.com/questions/43547402/how-to-calculate-f1-macro-in-keras\n    '''\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\n# https://ai.google/research/pubs/pub46697\nadam = optimizers.Adam()\nprint('LR:',K.eval(adam.lr))\n# 0.001 = learning rate in adam\n# optimal batch size ~ eps *N, where eps = learning rate and N = training size\nbatch_size = int(x_train.shape[0]*K.eval(adam.lr))\nprint('Batch size = ',batch_size)\n\n\n# Yoon Kim's shallow cnn model: https://arxiv.org/pdf/1408.5882.pdf\n# https://github.com/raghakot/keras-text/blob/master/keras_text/models/sequence_encoders.py\n#overfitting:\n# https://stackoverflow.com/questions/43156397/how-to-avoid-overfitting-in-the-given-convnet\n# http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/\n# https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n#word2vec    \ninp1 = Input(shape=(max_len,))\n#pretrained emb\nemb_layers = []\nmatrixes = [embedding_matrix,embedding_matrix_glov,embedding_matrix_wiki,embedding_matrix_para]\nfor matrix in matrixes:\n    emb_layers.append(Embedding(num_words, dim, weights=[matrix],trainable = False,)(inp1))\n\nx = Average()(emb_layers)\nx = SpatialDropout1D(0.1)(x)    \n\npooled_tensors = []\nkernel_sizes = [3,4,5]\nfor kernel_size in kernel_sizes:    \n    cnn = Conv1D(64, kernel_size=kernel_size, activation='relu',kernel_regularizer=regularizers.l2(0.0001))(x)\n    cnn = BatchNormalization()(cnn)\n    cnn = GlobalMaxPooling1D()(cnn)\n    pooled_tensors.append(cnn)\n\nif len(kernel_sizes) > 1:\n    x = concatenate(pooled_tensors, axis=-1)\nelse:\n    x = pooled_tensors[0]\n\n#classification dense net\nx = BatchNormalization()(x)\nx = Dropout(0.1)(x)\nx = Dense(192, activation=\"relu\",kernel_regularizer=regularizers.l2(0.0001))(x)\nx = BatchNormalization()(x)\nx = Dropout(0.2)(x)\nx = Dense(1, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001))(x)\n\n\nmodel = Model(inputs=inp1, outputs=x)\n\nmodel.compile(loss='binary_crossentropy', optimizer=adam, metrics=['accuracy',f1])\n\nprint(model.summary())\npatience = 5\n# for commiting the model to competition i need to comment these sections....otherwise the running time will be more then 2h on gpu...\nhistory = model.fit(x_train,y_train, \n                      batch_size=batch_size, \n                      validation_split=0.33,\n                      epochs=100,\n                      #overfits rather soon\n                      callbacks=[EarlyStopping(patience=patience,monitor='f1')])\n\nprint('training finished...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da921fc2d216b1e2db6819ea13ab0b0c4a0d731d"},"cell_type":"code","source":"_, ax = plt.subplots(1, 3, figsize=(20, 8))\nax[0].plot(history.history['loss'], label='loss')\nax[0].plot(history.history['val_loss'], label='val_loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['acc'], label='acc')\nax[1].plot(history.history['val_acc'], label='val_acc')\nax[1].legend()\nax[1].set_title('acc')\n\n\nax[2].plot(history.history['f1'], label='f1')\nax[2].plot(history.history['val_f1'], label='val_f1')\nax[2].legend()\nax[2].set_title('f1')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cde59508feb4a5c5f398128f9a5176e9a1912236","scrolled":true},"cell_type":"code","source":"#for train set\ny_pred = model.predict(x_train,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_train, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TRAINING SET:\\n',classification_report(y_train,y_pred))\n\n\n#for test set\ny_pred = model.predict(x_test,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_test, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TEST SET:\\n',classification_report(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3beb75a2101c603f182934220444bf573647220c"},"cell_type":"code","source":"print('fiting final model...')\nn_epochs = len(history.history['loss']) - patience\nhistory = model.fit(x_train,y_train, batch_size=batch_size, epochs=n_epochs)\n\nprint('fitting on full data done...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49e1bd5b69b428e57ad8248868d4b23c9c6f9d84"},"cell_type":"code","source":"_, ax = plt.subplots(1, 2, figsize=(12, 6))\nax[0].plot(history.history['loss'], label='loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['f1'], label='f1')\nax[1].legend()\nax[1].set_title('f1')\n\nplt.show()\n\ny_pred = model.predict(x_test,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_test, y_pred)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint(classification_report(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"#submission\nprint('Loading test data...')\ndf_final = pd.read_csv('../input/test.csv')\ndf_final[\"question_text\"].fillna(\"_##_\", inplace=True)\n\nx_final=tokenizer.texts_to_sequences(df_final['question_text'])\nx_final = pad_sequences(x_final,maxlen=max_len)\n\ny_pred = model.predict(x_final,batch_size=batch_size,verbose=1)\ny_pred = y_pred > search_result['threshold']\ny_pred = y_pred.astype(int)\nprint(y_pred[:5])\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = df_final.qid\ndf_subm['prediction'] = y_pred\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}