{"cells":[{"metadata":{"_uuid":"38e7d605b52588dfa82fb54def70d25e511df5bd"},"cell_type":"markdown","source":"### Inspired by:\n* https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n* https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\n* http://mlexplained.com/2018/01/13/weight-normalization-and-layer-normalization-explained-normalization-in-deep-learning-part-2/\n* https://arxiv.org/abs/1607.06450\n* https://github.com/keras-team/keras/issues/3878\n* https://www.kaggle.com/lystdo/lstm-with-word2vec-embeddings\n* https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\n* https://www.kaggle.com/aquatic/entity-embedding-neural-net\n* https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate\n* https://ai.google/research/pubs/pub46697\n* https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n\n\n(and other links in notebook)\n\nRemark:\nmodel overfits like hell...\n\nv6.1:\nincreased size of conv from 32 -> 100\nFor commit I have to disable training and tuning stage and fit on whole model, otherwise the running time is longer than 2 hours.\n\nv10:\n- use only one model\n- add lstm to cnn\n\nv11:\ntime distributed"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nnp.set_printoptions(threshold=np.nan)\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300\"))\n\n# Any results you write to the current directory are saved as output.\n\nimport gensim\nfrom gensim.utils import simple_preprocess\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report,f1_score,precision_recall_fscore_support,recall_score,precision_score\nfrom keras import backend as K\nfrom sklearn.utils import class_weight\nimport matplotlib.pyplot as plt\n\n#https://www.kaggle.com/shujian/single-rnn-with-4-folds-v1-9\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        print('\\rthreshold = %f | score = %f'%(threshold,score),end='')\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    print('\\nbest threshold is % f with score %f'%(best_threshold,best_score))\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77b7c5199c59943744495e62d7c0f73f68769e17"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf[\"question_text\"].fillna(\"_##_\",inplace=True)\nmax_len = df['question_text'].apply(lambda x:len(x)).max()\nprint('max length of sequences:',max_len)\n# df = df.sample(frac=0.1)\n\nprint('columns:',df.columns)\npd.set_option('display.max_columns',None)\nprint('df head:',df.head())\nprint('example of the question text values:',df['question_text'].head().values)\nprint('what values contains target:',df.target.unique())\n\nprint('Computing class weights....')\n#https://datascience.stackexchange.com/questions/13490/how-to-set-class-weights-for-imbalanced-classes-in-keras\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(df.target.values),\n                                                 df.target.values)\nprint('class_weights:',class_weights)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ff20343ef8bff4b01bd2940951dfb07210ec827"},"cell_type":"code","source":"print('removing stop words...')\n# remove stop words\n# https://www.kaggle.com/c/spooky-author-identification/discussion/42289\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import RegexpTokenizer\n\nregex_tokenizer = RegexpTokenizer(r'\\w+')\nstops=set(stopwords.words('english'))\nprint('number of stop words:',len(stops))\n\ndef delstopwords(sentence):\n    return \" \".join([word for word in regex_tokenizer.tokenize(sentence) if word not in stops])\n\ndf['question_text']=df['question_text'].apply(delstopwords)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f41b23c1f3f4eed0d8d419974fe795b63f3df50b"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n#dim of vectors\ndim = 300\n# max words in vocab\nnum_words = 50000\n# max number in questions\nmax_len = 100 \n\nprint('Fiting tokenizer')\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=num_words)\ntokenizer.fit_on_texts(df['question_text'])\n\nprint('spliting data')\ndf_train,df_test = train_test_split(df, random_state=1)\n\nprint('text to sequence')\nx_train = tokenizer.texts_to_sequences(df_train['question_text'])\nx_test = tokenizer.texts_to_sequences(df_test['question_text'])\n\nprint('pad sequence')\n## Pad the sentences \nx_train = pad_sequences(x_train,maxlen=max_len)\nx_test = pad_sequences(x_test, maxlen=max_len)\n\n## Get the target values\ny_train = df_train['target'].values\ny_test = df_test['target'].values\n\nprint(x_train.shape)\nprint(y_train.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db71c4ce3c1e3743f964c1a6e43a11644ee53cb4","scrolled":true},"cell_type":"code","source":"# https://www.kaggle.com/jhoward/improved-lstm-baseline-glove-dropout\nprint('loading word2vec model...')\nword2vec = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)\nprint('vocab:',len(word2vec.vocab))\n\nall_embs = word2vec.vectors\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nprint(emb_mean,emb_std)\n\nprint(num_words,' from ',len(tokenizer.word_index.items()))\n# num_words = min(num_words, len(tokenizer.word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (num_words, dim))\n\n# embedding_matrix = np.zeros((num_words, dim))\ncount = 0\nfor word, i in tokenizer.word_index.items():\n    if i>=num_words:\n        break\n    if word in word2vec.vocab:\n        embedding_matrix[i] = word2vec.word_vec(word)\n    else:\n        count += 1\nprint('embedding matrix size:',embedding_matrix.shape)\nprint('Number of words not in vocab:',count)\n\ndel word2vec\nimport gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55cb2ac1dca7de9fba51e8a7e5dba402159be302","scrolled":true},"cell_type":"code","source":"from keras.layers import Dense, Input,Embedding, Dropout, Activation, CuDNNLSTM,BatchNormalization,concatenate,SpatialDropout1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, Concatenate, GlobalAveragePooling1D,Average,Conv1D,GlobalMaxPooling1D,AlphaDropout\nfrom keras.layers import MaxPooling1D,UpSampling1D,RepeatVector,LSTM,TimeDistributed,Flatten\nfrom keras.models import Model\nfrom keras.callbacks import Callback,EarlyStopping\nfrom keras.engine import Layer\nfrom keras.initializers import Ones, Zeros\nimport keras.backend as K\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras import optimizers\n\ndef f1(y_true, y_pred):\n    '''\n    metric from here \n    https://stackoverflow.com/questions/43547402/how-to-calculate-f1-macro-in-keras\n    '''\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\n# https://ai.google/research/pubs/pub46697\nadam = optimizers.Adam()\nprint('LR:',K.eval(adam.lr))\n# 0.001 = learning rate in adam\n# optimal batch size ~ eps *N, where eps = learning rate and N = training size\nbatch_size = int(x_train.shape[0]*K.eval(adam.lr))\nprint('Batch size = ',batch_size)\n\n\n# Yoon Kim's shallow cnn model: https://arxiv.org/pdf/1408.5882.pdf\n# https://github.com/raghakot/keras-text/blob/master/keras_text/models/sequence_encoders.py\n#overfitting:\n# https://stackoverflow.com/questions/43156397/how-to-avoid-overfitting-in-the-given-convnet\n# http://www.wildml.com/2015/12/implementing-a-cnn-for-text-classification-in-tensorflow/\n# https://blog.openai.com/quantifying-generalization-in-reinforcement-learning/\n#word2vec    \ninp1 = Input(shape=(max_len,))\n# emb layer adds the most params to net...it's responsible for overfit? let's try setting them to non-trainable...\n# so it can be trainable, but needs to be regularized, otherwise it contributes to overfiting\n# e1 = Embedding(num_words, dim, weights=[embedding_matrix],trainable = True,embeddings_regularizer=regularizers.l2(0.0001))(inp1)\n# x = TimeDistributed(Dense(300,activation='relu',kernel_regularizer=regularizers.l2(0.0001)))(e1)\n# x = GlobalMaxPooling1D()(x)\n# e1 = BatchNormalization()(e1)\n# e1 = Dropout(0.1)(e1)\n# e1 = Dense(300, activation=\"relu\",kernel_regularizer=regularizers.l2(0.0001))(e1)\n# x = BatchNormalization()(x)\n# x = Dropout(0.2)(x)\n# x = Dense(1, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001))(x)\n\n#pretrained emb\ne1 = Embedding(num_words, dim, weights=[embedding_matrix],trainable = False,)(inp1)\n\n# #memory/encoder\n# x = CuDNNLSTM(max_len)(e1)\n# # x = GlobalMaxPooling1D()(x)\n\n#classification dense net\nx = Flatten()(e1)\nx = Dense(50, activation=\"relu\",)(x)\nx = Dropout(0.2)(x)\nx = Dense(1, activation='sigmoid',kernel_regularizer=regularizers.l2(0.0001))(x)\n\n\nmodel = Model(inputs=inp1, outputs=x)\n\nmodel.compile(loss='binary_crossentropy', optimizer=adam, metrics=[f1])\n\nprint(model.summary())\n\n# for commiting the model to competition i need to comment these sections....otherwise the running time will be more then 2h on gpu...\nhistory = model.fit(x_train,y_train, \n                      batch_size=batch_size, \n                      validation_split=0.33,\n                      epochs=100,\n                      #overfits rather soon\n                      callbacks=[EarlyStopping(patience=5,monitor='val_f1')])\n\nprint('training finished...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"da921fc2d216b1e2db6819ea13ab0b0c4a0d731d"},"cell_type":"code","source":"_, ax = plt.subplots(1, 2, figsize=(12, 6))\nax[0].plot(history.history['loss'], label='loss')\nax[0].plot(history.history['val_loss'], label='val_loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['f1'], label='f1')\nax[1].plot(history.history['val_f1'], label='val_f1')\nax[1].legend()\nax[1].set_title('f1')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cde59508feb4a5c5f398128f9a5176e9a1912236","scrolled":true},"cell_type":"code","source":"#for train set\ny_pred = model.predict(x_train,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_train, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TRAINING SET:\\n',classification_report(y_train,y_pred))\n\n\n#for test set\ny_pred = model.predict(x_test,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_test, y_pred)\nprint(search_result)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint('RESULTS ON TEST SET:\\n',classification_report(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3beb75a2101c603f182934220444bf573647220c"},"cell_type":"code","source":"print('fiting final model...')\n# sadly overfits, so 2 epochs top\nhistory = model.fit(x_train,y_train, batch_size=batch_size, epochs=6)\n\nprint('fitting on full data done...')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"49e1bd5b69b428e57ad8248868d4b23c9c6f9d84"},"cell_type":"code","source":"_, ax = plt.subplots(1, 2, figsize=(12, 6))\nax[0].plot(history.history['loss'], label='loss')\nax[0].legend()\nax[0].set_title('loss')\n\nax[1].plot(history.history['f1'], label='f1')\nax[1].legend()\nax[1].set_title('f1')\n\nplt.show()\n\ny_pred = model.predict(x_test,batch_size=batch_size, verbose=1)\nsearch_result = threshold_search(y_test, y_pred)\ny_pred = y_pred>search_result['threshold']\ny_pred = y_pred.astype(int)\n\nprint(classification_report(y_test,y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ad34df4ae095e1de08c64079fa6b0ecbc944423"},"cell_type":"code","source":"#submission\nprint('Loading test data...')\ndf_final = pd.read_csv('../input/test.csv')\ndf_final[\"question_text\"].fillna(\"_##_\", inplace=True)\n\nx_final=tokenizer.texts_to_sequences(df_final['question_text'])\nx_final = pad_sequences(x_final,maxlen=max_len)\n\ny_pred = model.predict(x_final,batch_size=batch_size,verbose=1)\ny_pred = y_pred > search_result['threshold']\ny_pred = y_pred.astype(int)\nprint(y_pred[:5])\n\ndf_subm = pd.DataFrame()\ndf_subm['qid'] = df_final.qid\ndf_subm['prediction'] = y_pred\nprint(df_subm.head())\ndf_subm.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}