{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from __future__ import absolute_import, division\nimport time\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n# !python -m spacy download en_core_web_lg\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input/embeddings\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import gensim\nfrom tqdm import tqdm\nfrom nltk.stem import PorterStemmer\nps = PorterStemmer()\nfrom nltk.stem.lancaster import LancasterStemmer\nlc = LancasterStemmer()\nfrom nltk.stem import SnowballStemmer\nsb = SnowballStemmer(\"english\")\nimport gc\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import LSTM, GRU\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nimport sys\nfrom os.path import dirname\n#sys.path.append(dirname(dirname(__file__)))\nfrom keras import initializers\nfrom keras.engine import InputSpec, Layer\nfrom keras import backend as K\n\nimport spacy","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()\nK.clear_session()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class AttentionWeightedAverage(Layer):\n    def __init__(self, return_attention=False, **kwargs):\n        self.init = initializers.get('uniform')\n        self.supports_masking = True\n        self.return_attention = return_attention\n        super(AttentionWeightedAverage, self).__init__(** kwargs)\n\n    def build(self, input_shape):\n        self.input_spec = [InputSpec(ndim=3)]\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight(shape=(input_shape[2], 1),\n                                 name='{}_W'.format(self.name),\n                                 initializer=self.init)\n        self.trainable_weights = [self.W]\n        super(AttentionWeightedAverage, self).build(input_shape)\n\n    def call(self, x, mask=None):\n        # computes a probability distribution over the timesteps\n        # uses 'max trick' for numerical stability\n        # reshape is done to avoid issue with Tensorflow\n        # and 1-dimensional weights\n        logits = K.dot(x, self.W)\n        x_shape = K.shape(x)\n        logits = K.reshape(logits, (x_shape[0], x_shape[1]))\n        ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))\n\n        # masked timesteps have zero weight\n        if mask is not None:\n            mask = K.cast(mask, K.floatx())\n            ai = ai * mask\n        att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon())\n        weighted_input = x * K.expand_dims(att_weights)\n        result = K.sum(weighted_input, axis=1)\n        if self.return_attention:\n            return [result, att_weights]\n        return result\n\n    def get_output_shape_for(self, input_shape):\n        return self.compute_output_shape(input_shape)\n\n    def compute_output_shape(self, input_shape):\n        output_len = input_shape[2]\n        if self.return_attention:\n            return [(input_shape[0], output_len), (input_shape[0], input_shape[1])]\n        return (input_shape[0], output_len)\n\n    def compute_mask(self, input, input_mask=None):\n        if isinstance(input_mask, list):\n            return [None] * len(input_mask)\n        else:\n            return None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"spell_model = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')\nwords = spell_model.index2word\nw_rank = {}\nfor i,word in enumerate(words):\n    w_rank[word] = i\nWORDS = w_rank","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del spell_model, w_rank\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use fast text as vocabulary\ndef words(text): return re.findall(r'\\w+', text.lower())\ndef P(word): \n    \"Probability of `word`.\"\n    # use inverse of rank as proxy\n    # returns 0 if the word isn't in the dictionary\n    return - WORDS.get(word, 0)\ndef correction(word): \n    \"Most probable spelling correction for word.\"\n    return max(candidates(word), key=P)\ndef candidates(word): \n    \"Generate possible spelling corrections for word.\"\n    return (known([word]) or known(edits1(word)) or [word])\ndef known(words): \n    \"The subset of `words` that appear in the dictionary of WORDS.\"\n    return set(w for w in words if w in WORDS)\ndef edits1(word):\n    \"All edits that are one edit away from `word`.\"\n    letters    = 'abcdefghijklmnopqrstuvwxyz'\n    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]\n    deletes    = [L + R[1:]               for L, R in splits if R]\n    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]\n    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]\n    inserts    = [L + c + R               for L, R in splits for c in letters]\n    return set(deletes + transposes + replaces + inserts)\ndef edits2(word): \n    \"All edits that are two edits away from `word`.\"\n    return (e2 for e1 in edits1(word) for e2 in edits1(e1))\ndef singlify(word):\n    return \"\".join([letter for i,letter in enumerate(word) if i == 0 or letter != word[i-1]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_glove(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef load_fasttext(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_para(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(embedding_matrix, nb_words, embedding_size=300):\n    inp = Input(shape=(max_length,))\n    x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.3)(x)\n    x1 = Bidirectional(LSTM(256, return_sequences=True))(x)\n    x2 = Bidirectional(GRU(128, return_sequences=True))(x1)\n    max_pool1 = GlobalMaxPooling1D()(x1)\n    max_pool2 = GlobalMaxPooling1D()(x2)\n    conc = Concatenate()([max_pool1, max_pool2])\n    predictions = Dense(1, activation='sigmoid')(conc)\n    model = Model(inputs=inp, outputs=predictions)\n    adam = optimizers.Adam(lr=learning_rate)\n    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Loading data ...\")\n\ntrain = pd.read_csv(\"../input/train.csv\").fillna(' ')\ntest = pd.read_csv('../input/test.csv').fillna(' ')\n\ntrain_text = train['question_text']\ntest_text = test['question_text']\ntext_list = pd.concat([train_text, test_text])\ny = train['target'].values\nnum_train_data = y.shape[0]\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Spacy NLP ...\")\nnlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])\nnlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)\nword_dict = {}\nword_index = 1\nlemma_dict = {}\ndocs = nlp.pipe(text_list, n_threads = 2)\nword_sequences = []\nfor doc in tqdm(docs):\n    word_seq = []\n    for token in doc:\n        if (token.text not in word_dict) and (token.pos_ is not \"PUNCT\"):\n            word_dict[token.text] = word_index\n            word_index += 1\n            lemma_dict[token.text] = token.lemma_\n        if token.pos_ is not \"PUNCT\":\n            word_seq.append(word_dict[token.text])\n    word_sequences.append(word_seq)\ndel docs\ngc.collect()\ntrain_word_sequences = word_sequences[:num_train_data]\ntest_word_sequences = word_sequences[num_train_data:]\n#################\ndel word_sequences\ngc.collect()\n######################\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\n# hyperparameters\nmax_length = 55\nembedding_size = 600\nlearning_rate = 0.001\nbatch_size = 512\nnum_epoch = 4\n\ntrain_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')\ntest_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')\n# print(train_word_sequences[:1])\n# print(test_word_sequences[:1])\npred_prob = np.zeros((len(test_word_sequences),), dtype=np.float32)\n\nstart_time = time.time()\nprint(\"Loading embedding matrix ...\")\nembedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)\nembedding_matrix_fasttext, nb_words = load_fasttext(word_dict, lemma_dict)\nembedding_matrix = np.concatenate((embedding_matrix_glove, embedding_matrix_fasttext), axis=1)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Start training ...\")\nstart_time = time.time()\nmodel = build_model(embedding_matrix, nb_words, embedding_size)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)\npred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)\npred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\ndel model, embedding_matrix_fasttext, embedding_matrix\ngc.collect()\nK.clear_session()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Loading embedding matrix ...\")\nembedding_matrix_para, nb_words = load_para(word_dict, lemma_dict)\nembedding_matrix = np.concatenate((embedding_matrix_glove, embedding_matrix_para), axis=1)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n############################\ndel embedding_matrix_para, embedding_matrix_glove, word_dict, lemma_dict\ngc.collect()\n# K.clear_session()\n###########################","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Start training ...\")\nmodel = build_model(embedding_matrix, nb_words, embedding_size)\n##########################\ndel embedding_matrix\ngc.collect()\n#################\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)\npred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\n\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)\npred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nsubmission = pd.DataFrame.from_dict({'qid': test['qid']})\nsubmission['prediction'] = (pred_prob>0.35).astype(int)\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}