{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from __future__ import absolute_import, division\n\nimport os\nimport time\nimport numpy as np\nimport pandas as pd\nimport gensim\nfrom tqdm import tqdm\nfrom nltk.stem import PorterStemmer\nps = PorterStemmer()\nfrom nltk.stem.lancaster import LancasterStemmer\nlc = LancasterStemmer()\nfrom nltk.stem import SnowballStemmer\nsb = SnowballStemmer(\"english\")\nimport gc\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nimport sys\nfrom os.path import dirname\n#sys.path.append(dirname(dirname(__file__)))\nfrom keras import initializers\nfrom keras.engine import InputSpec, Layer\nfrom keras import backend as K\n\nimport spacy\n\n# https://github.com/bfelbo/DeepMoji/blob/master/deepmoji/attlayer.py\nclass AttentionWeightedAverage(Layer):\n    \"\"\"\n    Computes a weighted average of the different channels across timesteps.\n    Uses 1 parameter pr. channel to compute the attention value for a single timestep.\n    \"\"\"\n\n    def __init__(self, return_attention=False, **kwargs):\n        self.init = initializers.get('uniform')\n        self.supports_masking = True\n        self.return_attention = return_attention\n        super(AttentionWeightedAverage, self).__init__(** kwargs)\n\n    def build(self, input_shape):\n        self.input_spec = [InputSpec(ndim=3)]\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight(shape=(input_shape[2], 1),\n                                 name='{}_W'.format(self.name),\n                                 initializer=self.init)\n        self.trainable_weights = [self.W]\n        super(AttentionWeightedAverage, self).build(input_shape)\n\n    def call(self, x, mask=None):\n        # computes a probability distribution over the timesteps\n        # uses 'max trick' for numerical stability\n        # reshape is done to avoid issue with Tensorflow\n        # and 1-dimensional weights\n        logits = K.dot(x, self.W)\n        x_shape = K.shape(x)\n        logits = K.reshape(logits, (x_shape[0], x_shape[1]))\n        ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))\n\n        # masked timesteps have zero weight\n        if mask is not None:\n            mask = K.cast(mask, K.floatx())\n            ai = ai * mask\n        att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon())\n        weighted_input = x * K.expand_dims(att_weights)\n        result = K.sum(weighted_input, axis=1)\n        if self.return_attention:\n            return [result, att_weights]\n        return result\n\n    def get_output_shape_for(self, input_shape):\n        return self.compute_output_shape(input_shape)\n\n    def compute_output_shape(self, input_shape):\n        output_len = input_shape[2]\n        if self.return_attention:\n            return [(input_shape[0], output_len), (input_shape[0], input_shape[1])]\n        return (input_shape[0], output_len)\n\n    def compute_mask(self, input, input_mask=None):\n        if isinstance(input_mask, list):\n            return [None] * len(input_mask)\n        else:\n            return None\n\n# https://www.kaggle.com/cpmpml/spell-checker-using-word2vec\nspell_model = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')\nwords = spell_model.index2word\nw_rank = {}\nfor i,word in enumerate(words):\n    w_rank[word] = i\nWORDS = w_rank\n# Use fast text as vocabulary\ndef words(text): return re.findall(r'\\w+', text.lower())\ndef P(word): \n    \"Probability of `word`.\"\n    # use inverse of rank as proxy\n    # returns 0 if the word isn't in the dictionary\n    return - WORDS.get(word, 0)\ndef correction(word): \n    \"Most probable spelling correction for word.\"\n    return max(candidates(word), key=P)\ndef candidates(word): \n    \"Generate possible spelling corrections for word.\"\n    return (known([word]) or known(edits1(word)) or [word])\ndef known(words): \n    \"The subset of `words` that appear in the dictionary of WORDS.\"\n    return set(w for w in words if w in WORDS)\ndef edits1(word):\n    \"All edits that are one edit away from `word`.\"\n    letters    = 'abcdefghijklmnopqrstuvwxyz'\n    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]\n    deletes    = [L + R[1:]               for L, R in splits if R]\n    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]\n    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]\n    inserts    = [L + c + R               for L, R in splits for c in letters]\n    return set(deletes + transposes + replaces + inserts)\ndef edits2(word): \n    \"All edits that are two edits away from `word`.\"\n    return (e2 for e1 in edits1(word) for e2 in edits1(e1))\ndef singlify(word):\n    return \"\".join([letter for i,letter in enumerate(word) if i == 0 or letter != word[i-1]])\n\n# modified version of \n# https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n# https://www.kaggle.com/danofer/different-embeddings-with-attention-fork\n# https://www.kaggle.com/shujian/different-embeddings-with-attention-fork-fork\ndef load_glove(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n\ndef load_fasttext(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n\ndef load_para(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n\ndef build_model(embedding_matrix, nb_words, embedding_size=300):\n    inp = Input(shape=(max_length,))\n    x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.3)(x)\n    x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)\n    x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x1)\n    max_pool1 = GlobalMaxPooling1D()(x1)\n    max_pool2 = GlobalMaxPooling1D()(x2)\n    conc = Concatenate()([max_pool1, max_pool2])\n    predictions = Dense(1, activation='sigmoid')(conc)\n    model = Model(inputs=inp, outputs=predictions)\n    adam = optimizers.Adam(lr=learning_rate)\n    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n    return model\n\nstart_time = time.time()\nprint(\"Loading data ...\")\ntrain = pd.read_csv('../input/train.csv').fillna(' ')\ntest = pd.read_csv('../input/test.csv').fillna(' ')\ntrain_text = train['question_text']\ntest_text = test['question_text']\ntext_list = pd.concat([train_text, test_text])\ny = train['target'].values\nnum_train_data = y.shape[0]\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Spacy NLP ...\")\nnlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])\nnlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)\nword_dict = {}\nword_index = 1\nlemma_dict = {}\ndocs = nlp.pipe(text_list, n_threads = 2)\nword_sequences = []\nfor doc in tqdm(docs):\n    word_seq = []\n    for token in doc:\n        if (token.text not in word_dict) and (token.pos_ is not \"PUNCT\"):\n            word_dict[token.text] = word_index\n            word_index += 1\n            lemma_dict[token.text] = token.lemma_\n        if token.pos_ is not \"PUNCT\":\n            word_seq.append(word_dict[token.text])\n    word_sequences.append(word_seq)\ndel docs\ngc.collect()\ntrain_word_sequences = word_sequences[:num_train_data]\ntest_word_sequences = word_sequences[num_train_data:]\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\n# hyperparameters\nmax_length = 55\nembedding_size = 600\nlearning_rate = 0.001\nbatch_size = 512\nnum_epoch = 4\n\ntrain_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')\ntest_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')\nprint(train_word_sequences[:1])\nprint(test_word_sequences[:1])\npred_prob = np.zeros((len(test_word_sequences),), dtype=np.float32)\n\nstart_time = time.time()\nprint(\"Loading embedding matrix ...\")\nembedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)\nembedding_matrix_fasttext, nb_words = load_fasttext(word_dict, lemma_dict)\nembedding_matrix = np.concatenate((embedding_matrix_glove, embedding_matrix_fasttext), axis=1)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Start training ...\")\nmodel = build_model(embedding_matrix, nb_words, embedding_size)\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)\npred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)\npred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\ndel model, embedding_matrix_fasttext, embedding_matrix\ngc.collect()\nK.clear_session()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Loading embedding matrix ...\")\nembedding_matrix_para, nb_words = load_para(word_dict, lemma_dict)\nembedding_matrix = np.concatenate((embedding_matrix_glove, embedding_matrix_para), axis=1)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Start training ...\")\nmodel = build_model(embedding_matrix, nb_words, embedding_size)\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)\npred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)\npred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nsubmission = pd.DataFrame.from_dict({'qid': test['qid']})\nsubmission['prediction'] = (pred_prob>0.35).astype(int)\nsubmission.to_csv('submission.csv', index=False)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}