{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport re\nimport collections\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom nltk.corpus import stopwords\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.utils.np_utils import to_categorical\nfrom sklearn.preprocessing import LabelEncoder\nfrom keras import models\nfrom keras import layers\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras import backend as K\nimport spacy\nfrom spacy.lang.en.stop_words import STOP_WORDS\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4153f44b98f6767ee6dfa9ead9a21c666332d8c6"},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37c158b19d7c8b89862fa37efde33da3068621db"},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14b313cddf01e6f79dea0993a23a5cbf883622ed"},"cell_type":"code","source":"#df = df.reindex(np.random.permutation(df.index))  \ntrain = train[['question_text', 'target']]\ntest_X = test[['question_text']]\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ceb81584e68301ad463a8f0561d07a247fbc4c3"},"cell_type":"code","source":"test_X.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48a4773038a1cfb75dda6758924de913b9a4afc5"},"cell_type":"code","source":"sns.countplot(x='target', data=train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02dff27fd48f4f2ce01becfbaf2e1bb48effb1fe"},"cell_type":"code","source":"NB_WORDS = 50000  # Parameter indicating the number of words we'll put in the dictionary\nVAL_SIZE = 1000  # Size of the validation set\nEPOCHS = 5  # Number of epochs we usually start to train with\nBATCH_SIZE = 1024  # Size of the batches used in the mini-batch gradient descent","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc48c875385b9d3205e431c2a3bd5c7b55054835"},"cell_type":"code","source":"stopwords_list = list(STOP_WORDS)\ndef remove_stopwords(input_text):\n        # Some words which might indicate a certain sentiment are kept via a whitelist\n        whitelist = [\"n't\", \"not\", \"no\"]\n        words = input_text.split() \n        clean_words = [word for word in words if (word not in stopwords_list or word in whitelist) and len(word) > 1] \n        return \" \".join(clean_words) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"412ce71033da05f3bcbbab2dc711d063686e7df2"},"cell_type":"code","source":"train.question_text = train.question_text.apply(remove_stopwords)\ntest_X.question_text = test_X.question_text.apply(remove_stopwords)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62e6d01b818dd62fd73f331302335f88747e6845"},"cell_type":"code","source":"X_train, X_valid, y_train, y_valid = train_test_split(train.question_text, train.target, test_size=0.1, \n                                                    random_state=37, stratify = train.target)\nprint('Training Data:', X_train.shape[0])\nprint('Validation Data:', X_valid.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1361122e78fdeb527bf4fb290fceb7c1f0d1c237"},"cell_type":"code","source":"tk = Tokenizer(num_words=NB_WORDS,\n               filters='!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n',\n               lower=True,\n               split=\" \")\ntk.fit_on_texts(X_train)\nprint('Top 5 most common words are:', collections.Counter(tk.word_counts).most_common(5))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"397807a1546f8c4786b25e11c795a0b403a71104"},"cell_type":"code","source":"word_index = tk.word_index\nprint('Found %s unique tokens.' % len(word_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42f1ef70a2388a0704891e6a9024b954f040ed69"},"cell_type":"code","source":"X_train_seq = tk.texts_to_sequences(X_train)\nX_valid_seq = tk.texts_to_sequences(X_valid)\nX_test_seq = tk.texts_to_sequences(test_X.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f38572e0e7f67ed2de15ba47385924054335fe2e"},"cell_type":"code","source":"len(X_test_seq)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ceeed3b974635efcbedf7020e9399769dfc38012"},"cell_type":"code","source":"seq_lengths = X_train.apply(lambda x: len(x.split(' ')))\nseq_lengths.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e1d62f979ea342f564edb2439f5c86225620b4d"},"cell_type":"code","source":"print('{} -- is converted to -- {}'.format(X_train[7], X_train_seq[7]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"41f315a48de66baad4d44d7369d0263e87d4275f"},"cell_type":"code","source":"MAX_LEN = 50\nX_train_seq_trunc = pad_sequences(X_train_seq, maxlen=MAX_LEN)\nX_valid_seq_trunc = pad_sequences(X_valid_seq, maxlen=MAX_LEN)\nX_test_seq_trunc = pad_sequences(X_test_seq, maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"92a811af1759ec7117d3b0ecb3fd039fc2034344"},"cell_type":"code","source":"print('{} -- is converted to -- {}'.format(X_train_seq[7], X_train_seq_trunc[7]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"acbe38f931d0664dd78d5ce3e6ad749efc50b6d4"},"cell_type":"code","source":"print('Shape of train set:',X_train_seq_trunc.shape)\nprint('Shape of validation set:',X_valid_seq_trunc.shape)\nprint('Shape of test set:',X_test_seq_trunc.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c74f1ac5ab0ce287a91ae22d836b00c20b397d5a"},"cell_type":"code","source":"# https://stackoverflow.com/questions/43547402/how-to-calculate-f1-macro-in-keras\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4f99ca518ec19c2c7bab841918593ef627c71493"},"cell_type":"code","source":"model = models.Sequential()\nmodel.add(layers.Embedding(NB_WORDS, 8, input_length=MAX_LEN))\nmodel.add(layers.CuDNNLSTM(64))\nmodel.add(layers.Dropout(0.2))\nmodel.add(layers.Dense(16, activation='relu'))\nmodel.add(layers.Dropout(0.2))\nmodel.add(layers.Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f76e5685cd26749a152ca2571e67070f9c2483ea"},"cell_type":"code","source":"#history = model.fit(X_train_seq_trunc, y_train, epochs=EPOCHS, batch_size=BATCH_SIZE, \n                    #validation_data=(X_valid_seq_trunc, y_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c976fde436085c6e6f0cf7cae188a79e7a9bfeaf"},"cell_type":"code","source":"#pred_noemb_y = model.predict([X_test_seq_trunc], batch_size=1024, verbose=1)\n#pred_noemb_y = (pred_noemb_y>0.35).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c699a205d8b8428f114a3191bdaf28de3cc55546"},"cell_type":"code","source":"#pred_noemb_y[0:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4633a24efcc2973889caf8a251d6e5a655f371a8"},"cell_type":"code","source":"#out_df = pd.DataFrame({\"qid\":test[\"qid\"].values})\n#out_df['prediction'] = pred_noemb_y\n#out_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14102fbc13ae4a8856e396743880d07cfc0b8432"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tk.word_index\nnb_words = min(NB_WORDS, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= NB_WORDS: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c3ec605d5021a7eaac8a97089505b67932f1d1c"},"cell_type":"code","source":"glove_model = models.Sequential()\nglove_model.add(layers.Embedding(NB_WORDS, embed_size, input_length=MAX_LEN, \n                                 weights=[embedding_matrix],trainable=False))\nglove_model.add(layers.CuDNNLSTM(64))\nglove_model.add(layers.Dropout(0.2))\nglove_model.add(layers.Dense(16, activation='relu'))\nglove_model.add(layers.Dropout(0.2))\nglove_model.add(layers.Dense(1, activation='sigmoid'))\nglove_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy', f1])\nglove_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"468f0132fcd2c28eb2c8aaa0a7f8eb578b582ef1"},"cell_type":"code","source":"history = glove_model.fit(X_train_seq_trunc, y_train, epochs=EPOCHS, batch_size=BATCH_SIZE, \n                    validation_data=(X_valid_seq_trunc, y_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07a9c9b018733acd5778027c0bd6ed646cb9aa4a"},"cell_type":"code","source":"pred_emb_y = glove_model.predict([X_test_seq_trunc], batch_size=1024, verbose=1)\npred_emb_y = (pred_emb_y>0.35).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d16d39e01e58c70d7cdd7681c0516c4860f9875b"},"cell_type":"code","source":"out_df = pd.DataFrame({\"qid\":test[\"qid\"].values})\nout_df['prediction'] = pred_emb_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be98b5c45c431dfb49bb44360cc11bfc646d3129"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}