{"cells":[{"metadata":{"trusted":true,"_uuid":"4bc24107e7b49e5f91dfdefcb2978f4722847eef"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom keras.models import Model, Sequential\nfrom keras.layers import Dense, Embedding, LSTM, Dropout, Bidirectional\nfrom keras.preprocessing import sequence\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.callbacks import Callback, EarlyStopping\nfrom keras import regularizers\nfrom sklearn.metrics import f1_score\nfrom gensim.models.keyedvectors import KeyedVectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5be19211537dd999f7ea8bcbbb61f75645165b11"},"cell_type":"code","source":"hyper_params = {\n    'validation_split': 0.01,\n    'batch_size': 64,\n    'sample_ratio': 1.0,\n    'num_words': 40000,\n    'epochs': 10,\n    'embedding_size': 300,\n    'keep_probability': 0.75,\n    'l2_regularization': 0.01,\n    'lstm_size': 50,\n    'dense_size': 50,\n    'max_sequence': 60,\n    'sampling_size': -1,\n    'min_length': 1,\n    'max_length': 50\n}\n\nconfig = {\n    'lowercase': True,\n    'stemming': True,\n    'remove_stopwords': True,\n    'remove_non_letters': True,\n    'remove_punctuation': True,\n    'reduce_lengthening': True,\n    'sort': True,\n    'trim': True,\n    'early_stopping': False,\n    'sub_sampling': True\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2513085dd22b79df19d221f4f06d0443aa268e92"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\nif hyper_params['sampling_size'] > 0:\n    train = train.sample(hyper_params['sampling_size'])\nif config['sub_sampling']:\n    negative_df = train[train['target'] == 0]\n    positive_df = train[train['target'] == 1]\n    positive = int((len(train) - len(negative_df)) * hyper_params['sample_ratio'])\n    train = pd.concat([negative_df.sample(positive, random_state=42), positive_df])\ntest = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6425922ec770f7da4a43c52fcddeca1643cc4fea"},"cell_type":"code","source":"import re\nimport string\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\n\ndef reduce_lengthening(data):\n    length_regex = re.compile(r\"(.)\\1{2,}\")\n    return [length_regex.sub(r\"\\1\\1\", x) for x in data['question_text']]\n\ndef remove_non_letters(data):\n    letters_regex = re.compile('[^a-zA-Z ]')\n    return [letters_regex.sub(' ', x) for x in data['question_text']]\n\ndef remove_stopwords(data):\n    stopwords_set = set(stopwords.words('english'))\n    temp = [[y if y not in stopwords_set else '' for y in x] for x in data['question_text']]\n    return [filter(None, x) for x in temp]\n\ndef stemming(data):\n    ps = PorterStemmer()\n    temp = [[str(ps.stem(y)) for y in x.split(' ')] for x in data['question_text']]\n    return [' '.join(x) for x in temp]\n\ndef remove_punctuation(data):\n    return [x.translate(string.punctuation) for x in data['question_text']]\n\ndef lowercase(data):\n    return [x.lower() for x in data['question_text']]\n\nif config['lowercase']:\n    train['question_text'] = lowercase(train)\n    test['question_text'] = lowercase(test)\nif config['remove_non_letters']:\n    train['question_text'] = remove_non_letters(train)\n    test['question_text'] = remove_non_letters(test)\nif config['reduce_lengthening']:\n    train['question_text'] = reduce_lengthening(train)\n    test['question_text'] = reduce_lengthening(test)\nif config['remove_punctuation']:\n    train['question_text'] = remove_punctuation(train)\n    test['question_text'] = remove_punctuation(test)\nif config['stemming']:\n    train['question_text'] = stemming(train)\n    test['question_text'] = stemming(test)\nif config['trim']:\n    train = train[train.apply(lambda x: len(x['question_text'].split(' ')) >= hyper_params['min_length'] and \n                              len(x['question_text'].split(' ')) <= hyper_params['max_length'], axis=1)]\nif config['sort']:\n    train = train.reindex(train.question_text.str.len().sort_values().index)\n    train = train.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63c6edb3ffbcc69640af712600f121721bfe402d"},"cell_type":"code","source":"train_sentences = train['question_text'].values\ntest_sentences = test['question_text'].values\ntokenizer = Tokenizer(num_words=hyper_params['num_words'])\ntokenizer.fit_on_texts(train_sentences)\ntrain_tokenized = tokenizer.texts_to_sequences(train_sentences)\ntest_tokenized = tokenizer.texts_to_sequences(test_sentences)\nword_index = tokenizer.word_index\nX_train = sequence.pad_sequences(train_tokenized, maxlen=hyper_params['max_sequence'])\ny_train = train.target.values\nX_test = sequence.pad_sequences(test_tokenized, maxlen=hyper_params['max_sequence'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f8e627f25f642d6f1140b09291b0c4aac5fd8508"},"cell_type":"code","source":"word_vectors = KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)\nvocabulary_size = min(len(word_index) + 1, hyper_params['num_words'])\nembedding_matrix = np.zeros((vocabulary_size, hyper_params['embedding_size']))\nfor word, i in word_index.items():\n    if i>= hyper_params['num_words']:\n        continue\n    try:\n        embedding_vector = word_vectors[word]\n        embedding_matrix[i] = embedding_vector\n    except KeyError:\n        embedding_matrix[i] = np.random.normal(0, np.sqrt(0.25), hyper_params['embedding_size'])\ndel(word_vectors)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0e23ddbd788829a8ece892db5a67251b910f7606"},"cell_type":"code","source":"class Metrics(Callback):\n    def on_epoch_end(self, epoch, logs={}):\n        val_predict = (np.asarray(self.model.predict(self.validation_data[0]))).round()\n        val_targ = self.validation_data[1]\n        print(\" — f1_score: %f\" % f1_score(val_targ, val_predict))\nmetrics = Metrics()\nearly_stopping = EarlyStopping(monitor='val_loss', patience=1, restore_best_weights=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7376282b438850d4a3f3d675eb0adbe519e90392"},"cell_type":"code","source":"model = Sequential()\nmodel.add(Embedding(vocabulary_size, hyper_params['embedding_size'], weights=[embedding_matrix], trainable=True))\nmodel.add(Bidirectional(LSTM(hyper_params['lstm_size'], dropout=1 - hyper_params['keep_probability'])))\nmodel.add(Dense(hyper_params['dense_size'], activation='relu', \n                kernel_regularizer=regularizers.l2(hyper_params['l2_regularization'])))\nmodel.add(Dropout(1 - hyper_params['keep_probability']))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['acc'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"311da9d253b06b489155097662509bf1707293bc"},"cell_type":"code","source":"callbacks = [metrics]\nif config['early_stopping']:\n    callbacks += early_stopping\nhistory = model.fit(X_train, y_train, batch_size=hyper_params['batch_size'], epochs=hyper_params['epochs'], validation_split=hyper_params['validation_split'], \n                    callbacks=callbacks)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5237a6e8a68cf45529edfa4d98a8f0b8f349420"},"cell_type":"code","source":"sample_submission = pd.read_csv('../input/sample_submission.csv')\nsample_submission.prediction = model.predict_classes(X_test)\nsample_submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}