{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"### Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n* https://ai.google/research/pubs/pub46697\n* https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n* https://www.kaggle.com/rasvob/let-s-try-clr-v3\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport sys\nnp.set_printoptions(threshold=sys.maxsize)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\nimport tensorflow as tf\n\n# SEED = 2018\n\n# np.random.seed(SEED)\n# tf.set_random_seed(SEED)\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    print('\\nbest threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77b7c5199c59943744495e62d7c0f73f68769e17"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf[\"question_text\"].fillna(\"_##_\",inplace=True)\nmax_len = df['question_text'].apply(lambda x:len(x)).max()\nprint('max length of sequences:',max_len)\n# df = df.sample(frac=0.1)\n\nprint('columns:',df.columns)\npd.set_option('display.max_columns',None)\nprint('df head:',df.head())\nprint('example of the question text values:',df['question_text'].head().values)\nprint('what values contains target:',df.target.unique())\n\nprint('Loading test data...')\ndf_final = pd.read_csv('../input/test.csv')\ndf_final[\"question_text\"].fillna(\"_##_\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f41b23c1f3f4eed0d8d419974fe795b63f3df50b"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#dim of vectors\ndim = 300\n# max words in vocab\nnum_words = 75966\n# max number in questions\nmax_len = 100 \n\nprint('Fiting tokenizer')\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=num_words)\ntokenizer.fit_on_texts(list(df['question_text'])+list(df_final['question_text']))\n\nprint('text to sequence')\nx_train = tokenizer.texts_to_sequences(df['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx_train = pad_sequences(x_train,maxlen=max_len)\n\n## Get the target values\ny_train = df['target'].values\n\nprint(x_train.shape)\nprint(y_train.shape)\n\nx_test=tokenizer.texts_to_sequences(df_final['question_text'])\nx_test = pad_sequences(x_test,maxlen=max_len)\n\nprint('Test data loaded:',x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f1ed31984c07cbb1a95c250e0dadf9eb649e5a3"},"cell_type":"code","source":"print('Glove ... ')\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt'))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n\nword_index = tokenizer.word_index\n# num_words = min(num_words, len(word_index))\nembedding_matrix_glov = np.random.normal(emb_mean, emb_std, (num_words, dim))\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_glov[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9712758dc3cab221d6e57f43e5eb00224386d7d5"},"cell_type":"code","source":"print('Para...')\nEMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(len(all_embs))\n\n\nword_index = tokenizer.word_index\n# num_words = min(num_words, len(word_index))\nembedding_matrix_para = np.random.normal(emb_mean, emb_std, (num_words, dim))\ncount=0\nfor word, i in word_index.items():\n    if i >= num_words: \n        break\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_para[i] = embedding_vector\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix_glov.shape)\nprint('Number of words not in vocab:',count)\n\ndel embeddings_index,all_embs\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d17e4323bef806ca6fa47c1edf8a3764a082caa3"},"cell_type":"code","source":"matrixes = [embedding_matrix_glov,embedding_matrix_para]\n\nmatrix = np.mean(matrixes,axis=0)\n\ndel embedding_matrix_glov,embedding_matrix_para\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55cb2ac1dca7de9fba51e8a7e5dba402159be302","scrolled":true},"cell_type":"code","source":"from keras.layers import Dense, Input,Embedding, Dropout, Activation, CuDNNLSTM,BatchNormalization,concatenate,SpatialDropout1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Concatenate, GlobalAveragePooling1D,Average,Conv1D,GlobalMaxPooling1D,AlphaDropout\nfrom keras.layers import MaxPooling1D,UpSampling1D,RepeatVector,LSTM,TimeDistributed,Flatten, CuDNNGRU, Add\nfrom keras.models import Model\nfrom keras.callbacks import Callback,EarlyStopping,ModelCheckpoint, ReduceLROnPlateau\nfrom keras.engine import Layer\nfrom keras.initializers import Ones, Zeros\nimport keras.backend as K\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras import optimizers\nfrom keras import initializers\nfrom sklearn.model_selection import KFold, StratifiedKFold\nimport warnings\n\ndef get_model(trainable=False):\n    inp1 = Input(shape=(max_len,))\n    emb = Embedding(num_words, dim, weights=[matrix],trainable = trainable,)(inp1)\n    x,h_f,c_f,h_b,c_b = Bidirectional(CuDNNLSTM(128,return_sequences=True,return_state=True))(emb)\n    x = concatenate([h_f,h_b])\n    x = Dense(128, activation=\"relu\")(x)\n    x = Dense(1, activation='sigmoid')(x)\n\n    model = Model(inputs=inp1, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n#     print(model.summary())\n    return model\n\nbatch_size = 512\nprint('Batch size = ',batch_size)\n\npatience = 2\n\nbest_model=None\nall_results = {}\n\ntrain_meta = np.zeros(y_train.shape)\ntest_meta = np.zeros(x_test.shape[0])\n\nsplits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=14).split(x_train, y_train))\n\nfor idx, (train_idx, valid_idx) in enumerate(splits):\n    print('----'+str(idx)+'-----')\n    X_train1 = x_train[train_idx]\n    y_train1 = y_train[train_idx]\n    X_val = x_train[valid_idx]\n    y_val = y_train[valid_idx]\n    model_file = 'model_'+str(idx)+'.h5'\n    modelcheck = ModelCheckpoint(model_file,save_best_only=True)\n    stop = EarlyStopping(patience=patience)\n    \n    model = get_model()\n    \n    history = model.fit(X_train1,y_train1, \n                      batch_size=batch_size, \n                      validation_data=(X_val,y_val),\n                      epochs=100,\n                      callbacks=[modelcheck,stop],\n                      verbose=2)\n#     print('Pretraining finished, unfreezing embeddings layer...')\n    \n#     model = get_model(trainable=True)\n#     model.load_weights(model_file)    \n    \n#     modelcheck = ModelCheckpoint(model_file,save_best_only=True)\n#     stop = EarlyStopping(patience=patience)\n#     metrics = Metrics()\n    \n#     history = model.fit(X_train1,y_train1, \n#                       batch_size=batch_size, \n#                       validation_data=(X_val,y_val),\n#                       epochs=1,\n#                       #overfits rather soon\n#                       callbacks=[modelcheck,stop],\n#                       verbose=2)\n    \n    \n    print('training finished...')\n\n    #load best performing\n    model.load_weights(model_file)\n\n    #for val set\n    y_pred = model.predict(X_val,batch_size=batch_size, verbose=1)\n    train_meta[valid_idx] = y_pred.reshape(-1)\n\n    search_result = threshold_search(y_val, y_pred)\n    print(search_result)\n    y_pred = y_pred>search_result['threshold']\n    y_pred = y_pred.astype(int)\n\n    print('RESULTS ON VALIDATION SET:\\n',classification_report(y_val,y_pred))\n\n    all_results[model_file] = search_result['f1']    \n    \n    #for test set\n    y_pred = model.predict(x_test,batch_size=batch_size, verbose=1)\n    test_meta += y_pred.reshape(-1) / len(splits)\n    \n    if best_model is None or best_model['f1']  < search_result['f1']:\n        best_model={'model':model_file,'f1':search_result['f1']}\n    \n    \nprint('-'*80)\nprint(all_results)\nprint('-'*80)\nprint(best_model)\nprint('-'*80)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"#submission\nsearch_result = threshold_search(y_train, train_meta)\nprint(search_result)\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = df_final.qid\ndf_subm['prediction'] = test_meta > search_result['threshold']\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}