{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-08-23T13:10:36.87462Z","iopub.execute_input":"2022-08-23T13:10:36.875355Z","iopub.status.idle":"2022-08-23T13:10:36.89025Z","shell.execute_reply.started":"2022-08-23T13:10:36.875244Z","shell.execute_reply":"2022-08-23T13:10:36.888907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_train=pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ndata_train","metadata":{"execution":{"iopub.status.busy":"2022-08-23T13:14:02.041088Z","iopub.execute_input":"2022-08-23T13:14:02.041496Z","iopub.status.idle":"2022-08-23T13:14:05.305101Z","shell.execute_reply.started":"2022-08-23T13:14:02.041456Z","shell.execute_reply":"2022-08-23T13:14:05.304017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from __future__ import absolute_import, division\n\nimport os\nimport time\nimport numpy as np\nimport pandas as pd\nimport gensim\nfrom tqdm import tqdm\nfrom nltk.stem import PorterStemmer\nps = PorterStemmer()\nfrom nltk.stem.lancaster import LancasterStemmer\nlc = LancasterStemmer()\nfrom nltk.stem import SnowballStemmer\nsb = SnowballStemmer(\"english\")\nimport gc\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nimport sys\nfrom os.path import dirname\n#sys.path.append(dirname(dirname(__file__)))\nfrom keras import initializers\nfrom keras.engine import InputSpec, Layer\nfrom keras import backend as K\n\nimport spacy\n\n# https://github.com/bfelbo/DeepMoji/blob/master/deepmoji/attlayer.py\nclass AttentionWeightedAverage(Layer):\n    \"\"\"\n    Computes a weighted average of the different channels across timesteps.\n    Uses 1 parameter pr. channel to compute the attention value for a single timestep.\n    \"\"\"\n\n    def __init__(self, return_attention=False, **kwargs):\n        self.init = initializers.get('uniform')\n        self.supports_masking = True\n        self.return_attention = return_attention\n        super(AttentionWeightedAverage, self).__init__(** kwargs)\n\n    def build(self, input_shape):\n        self.input_spec = [InputSpec(ndim=3)]\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight(shape=(input_shape[2], 1),\n                                 name='{}_W'.format(self.name),\n                                 initializer=self.init)\n        self.trainable_weights = [self.W]\n        super(AttentionWeightedAverage, self).build(input_shape)\n\n    def call(self, x, mask=None):\n        # computes a probability distribution over the timesteps\n        # uses 'max trick' for numerical stability\n        # reshape is done to avoid issue with Tensorflow\n        # and 1-dimensional weights\n        logits = K.dot(x, self.W)\n        x_shape = K.shape(x)\n        logits = K.reshape(logits, (x_shape[0], x_shape[1]))\n        ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))\n\n        # masked timesteps have zero weight\n        if mask is not None:\n            mask = K.cast(mask, K.floatx())\n            ai = ai * mask\n        att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon())\n        weighted_input = x * K.expand_dims(att_weights)\n        result = K.sum(weighted_input, axis=1)\n        if self.return_attention:\n            return [result, att_weights]\n        return result\n\n    def get_output_shape_for(self, input_shape):\n        return self.compute_output_shape(input_shape)\n\n    def compute_output_shape(self, input_shape):\n        output_len = input_shape[2]\n        if self.return_attention:\n            return [(input_shape[0], output_len), (input_shape[0], input_shape[1])]\n        return (input_shape[0], output_len)\n\n    def compute_mask(self, input, input_mask=None):\n        if isinstance(input_mask, list):\n            return [None] * len(input_mask)\n        else:\n            return None\n\n# https://www.kaggle.com/cpmpml/spell-checker-using-word2vec\nspell_model = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec')\nwords = spell_model.index2word\nw_rank = {}\nfor i,word in enumerate(words):\n    w_rank[word] = i\nWORDS = w_rank\n# Use fast text as vocabulary\ndef words(text): return re.findall(r'\\w+', text.lower())\ndef P(word): \n    \"Probability of `word`.\"\n    # use inverse of rank as proxy\n    # returns 0 if the word isn't in the dictionary\n    return - WORDS.get(word, 0)\ndef correction(word): \n    \"Most probable spelling correction for word.\"\n    return max(candidates(word), key=P)\ndef candidates(word): \n    \"Generate possible spelling corrections for word.\"\n    return (known([word]) or known(edits1(word)) or [word])\ndef known(words): \n    \"The subset of `words` that appear in the dictionary of WORDS.\"\n    return set(w for w in words if w in WORDS)\ndef edits1(word):\n    \"All edits that are one edit away from `word`.\"\n    letters    = 'abcdefghijklmnopqrstuvwxyz'\n    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]\n    deletes    = [L + R[1:]               for L, R in splits if R]\n    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]\n    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]\n    inserts    = [L + c + R               for L, R in splits for c in letters]\n    return set(deletes + transposes + replaces + inserts)\ndef edits2(word): \n    \"All edits that are two edits away from `word`.\"\n    return (e2 for e1 in edits1(word) for e2 in edits1(e1))\ndef singlify(word):\n    return \"\".join([letter for i,letter in enumerate(word) if i == 0 or letter != word[i-1]])\n\n# modified version of \n# https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n# https://www.kaggle.com/danofer/different-embeddings-with-attention-fork\n# https://www.kaggle.com/shujian/different-embeddings-with-attention-fork-fork\ndef load_glove(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n\ndef load_fasttext(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n\ndef load_para(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n\ndef build_model(embedding_matrix, nb_words, embedding_size=300):\n    inp = Input(shape=(max_length,))\n    x = Embedding(nb_words, embedding_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.3)(x)\n    x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)\n    x2 = Bidirectional(CuDNNGRU(128, return_sequences=True))(x1)\n    max_pool1 = GlobalMaxPooling1D()(x1)\n    max_pool2 = GlobalMaxPooling1D()(x2)\n    conc = Concatenate()([max_pool1, max_pool2])\n    predictions = Dense(1, activation='sigmoid')(conc)\n    model = Model(inputs=inp, outputs=predictions)\n    adam = optimizers.Adam(lr=learning_rate)\n    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n    return model\n\nstart_time = time.time()\nprint(\"Loading data ...\")\ntrain = pd.read_csv('../input/train.csv').fillna(' ')\ntest = pd.read_csv('../input/test.csv').fillna(' ')\ntrain_text = train['question_text']\ntest_text = test['question_text']\ntext_list = pd.concat([train_text, test_text])\ny = train['target'].values\nnum_train_data = y.shape[0]\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Spacy NLP ...\")\nnlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])\nnlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)\nword_dict = {}\nword_index = 1\nlemma_dict = {}\ndocs = nlp.pipe(text_list, n_threads = 2)\nword_sequences = []\nfor doc in tqdm(docs):\n    word_seq = []\n    for token in doc:\n        if (token.text not in word_dict) and (token.pos_ is not \"PUNCT\"):\n            word_dict[token.text] = word_index\n            word_index += 1\n            lemma_dict[token.text] = token.lemma_\n        if token.pos_ is not \"PUNCT\":\n            word_seq.append(word_dict[token.text])\n    word_sequences.append(word_seq)\ndel docs\ngc.collect()\ntrain_word_sequences = word_sequences[:num_train_data]\ntest_word_sequences = word_sequences[num_train_data:]\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\n# hyperparameters\nmax_length = 55\nembedding_size = 600\nlearning_rate = 0.001\nbatch_size = 512\nnum_epoch = 4\n\ntrain_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')\ntest_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')\nprint(train_word_sequences[:1])\nprint(test_word_sequences[:1])\npred_prob = np.zeros((len(test_word_sequences),), dtype=np.float32)\n\nstart_time = time.time()\nprint(\"Loading embedding matrix ...\")\nembedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)\nembedding_matrix_fasttext, nb_words = load_fasttext(word_dict, lemma_dict)\nembedding_matrix = np.concatenate((embedding_matrix_glove, embedding_matrix_fasttext), axis=1)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Start training ...\")\nmodel = build_model(embedding_matrix, nb_words, embedding_size)\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)\npred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)\npred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\ndel model, embedding_matrix_fasttext, embedding_matrix\ngc.collect()\nK.clear_session()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Loading embedding matrix ...\")\nembedding_matrix_para, nb_words = load_para(word_dict, lemma_dict)\nembedding_matrix = np.concatenate((embedding_matrix_glove, embedding_matrix_para), axis=1)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Start training ...\")\nmodel = build_model(embedding_matrix, nb_words, embedding_size)\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)\npred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)\npred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nsubmission = pd.DataFrame.from_dict({'qid': test['qid']})\nsubmission['prediction'] = (pred_prob>0.35).astype(int)\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-08-23T13:10:42.190398Z","iopub.execute_input":"2022-08-23T13:10:42.190879Z","iopub.status.idle":"2022-08-23T13:10:51.488859Z","shell.execute_reply.started":"2022-08-23T13:10:42.19081Z","shell.execute_reply":"2022-08-23T13:10:51.48699Z"},"trusted":true},"execution_count":null,"outputs":[]}]}