{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For examplye, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from __future__ import absolute_import, division\n\nimport os\nimport time\nimport numpy as np\nimport pandas as pd\nimport gensim\nfrom tqdm import tqdm\nfrom nltk.stem import PorterStemmer\nps = PorterStemmer()\nfrom nltk.stem.lancaster import LancasterStemmer\nlc = LancasterStemmer()\nfrom nltk.stem import SnowballStemmer\nsb = SnowballStemmer(\"english\")\nimport gc\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nimport sys\nfrom os.path import dirname\n#sys.path.append(dirname(dirname(__file__)))\nfrom keras import initializers\nfrom keras.engine import InputSpec, Layer\nfrom keras import backend as K\n\nimport spacy\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://www.kaggle.com/cpmpml/spell-checker-using-word2vec\n\n# Use fast text as vocabulary\ndef words(text): return re.findall(r'\\w+', text.lower())\ndef P(word): \n    \"Probability of `word`.\"\n    # use inverse of rank as proxy\n    # returns 0 if the word isn't in the dictionary\n    return - WORDS.get(word, 0)\ndef correction(word): \n    \"Most probable spelling correction for word.\"\n    return max(candidates(word), key=P)\ndef candidates(word): \n    \"Generate possible spelling corrections for word.\"\n    return (known([word]) or known(edits1(word)) or [word])\ndef known(words): \n    \"The subset of `words` that appear in the dictionary of WORDS.\"\n    return set(w for w in words if w in WORDS)\ndef edits1(word):\n    \"All edits that are one edit away from `word`.\"\n    letters    = 'abcdefghijklmnopqrstuvwxyz'\n    splits     = [(word[:i], word[i:])    for i in range(len(word) + 1)]\n    deletes    = [L + R[1:]               for L, R in splits if R]\n    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]\n    replaces   = [L + c + R[1:]           for L, R in splits if R for c in letters]\n    inserts    = [L + c + R               for L, R in splits for c in letters]\n    return set(deletes + transposes + replaces + inserts)\ndef edits2(word): \n    \"All edits that are two edits away from `word`.\"\n    return (e2 for e1 in edits1(word) for e2 in edits1(e1))\ndef singlify(word):\n    return \"\".join([letter for i,letter in enumerate(word) if i == 0 or letter != word[i-1]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# modified version of \n# https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n# https://www.kaggle.com/danofer/different-embeddings-with-attention-fork\n# https://www.kaggle.com/shujian/different-embeddings-with-attention-fork-fork\ndef load_glove(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/glove840b300dtxt/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n\ndef load_fasttext(word_dict, lemma_dict):\n    EMBEDDING_FILE = '../input/fasttext-crawl-300d-2m/crawl-300d-2M.vec'\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.strip().split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n    embed_size = 300\n    nb_words = len(word_dict)+1\n    embedding_matrix = np.zeros((nb_words, embed_size), dtype=np.float32)\n    unknown_vector = np.zeros((embed_size,), dtype=np.float32) - 1.\n    print(unknown_vector[:5])\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.lower()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.upper()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = key.capitalize()\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = ps.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lc.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = sb.stem(key)\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        word = lemma_dict[key]\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None:\n                embedding_matrix[word_dict[key]] = embedding_vector\n                continue\n        embedding_matrix[word_dict[key]] = unknown_vector                    \n    return embedding_matrix, nb_words \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\ndef build_model_simplelstm(embedding_matrix, num_aux_targets):\n    words = Input(shape=(None,))\n    x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False)(words)\n    x = SpatialDropout1D(0.3)(x)\n    x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)\n    x = Bidirectional(CuDNNLSTM(LSTM_UNITS, return_sequences=True))(x)\n\n    hidden = concatenate([\n        GlobalMaxPooling1D()(x),\n        GlobalAveragePooling1D()(x),\n    ])\n    hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu')(hidden)])\n    hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu')(hidden)])\n    result = Dense(1, activation='sigmoid')(hidden)\n    aux_result = Dense(num_aux_targets, activation='sigmoid')(hidden)\n    \n    model = Model(inputs=words, outputs=[result, aux_result])\n    model.compile(loss='binary_crossentropy', optimizer='adam' ,  metrics=['accuracy'] )\n\n    return model\n\ndef build_model2(embedding_matrix, num_aux_targets):\n    words = Input(shape=(None,))\n    x = Embedding(*embedding_matrix.shape, weights=[embedding_matrix], trainable=False)(words)\n    x = SpatialDropout1D(0.3)(x)\n    x1 = Bidirectional(CuDNNLSTM(256, return_sequences=True))(x)\n    x2 = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n\n    hidden = concatenate([\n        GlobalMaxPooling1D()(x1),\n        GlobalMaxPooling1D()(x2),\n    ])\n    \n    hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu')(hidden)])\n    hidden = add([hidden, Dense(DENSE_HIDDEN_UNITS, activation='relu')(hidden)])\n    result = Dense(1, activation='sigmoid')(hidden)\n    aux_result = Dense(num_aux_targets, activation='sigmoid')(hidden)\n    \n    model = Model(inputs=words, outputs=[result, aux_result])\n    model.compile(loss='binary_crossentropy', optimizer='adam' , metrics=['accuracy'] )\n\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Loading data ...\")\ntrain = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/train.csv')\ntest = pd.read_csv('../input/jigsaw-unintended-bias-in-toxicity-classification/test.csv')\n#train_text = train['comment_text']\n#test_text = test['comment_text']\n#text_list = pd.concat([train_text, test_text])\n#num_train_data = y.shape[0]\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = train.head(1000000)\n#test = test.head(1000000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nstart_time = time.time()\nprint(\"Spacy NLP ...\")\nnlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])\nnlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)\nword_dict = {}\nword_index = 1\nlemma_dict = {}\ndocs = nlp.pipe(pd.concat([train['comment_text'], test['comment_text']]), n_threads = 2)\nword_sequences = []\nfor doc in tqdm(docs):\n    word_seq = []\n    for token in doc:\n        if (token.text not in word_dict) and (token.pos_ is not \"PUNCT\"):\n            word_dict[token.text] = word_index\n            word_index += 1\n            lemma_dict[token.text] = token.lemma_\n        if token.pos_ is not \"PUNCT\":\n            word_seq.append(word_dict[token.text])\n    word_sequences.append(word_seq)\ndel docs\ngc.collect()\nnum_train_data = len(train)\ntrain_word_sequences = word_sequences[:num_train_data]\ntest_word_sequences = word_sequences[num_train_data:]\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del nlp\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Build spell correction dictnary ...\")\nspell_model = gensim.models.KeyedVectors.load_word2vec_format('../input/wikinews300d1mvec/wiki-news-300d-1M.vec')\nwords = spell_model.index2word\nw_rank = {}\nfor i,word in enumerate(words):\n    w_rank[word] = i\nWORDS = w_rank\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del spell_model\ndel words\ndel w_rank\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Loading embedding matrix ...\")\nembedding_matrix_glove, nb_words = load_glove(word_dict, lemma_dict)\n#embedding_matrix_fasttext, nb_words = load_fasttext(word_dict, lemma_dict)\n#embedding_matrix = np.concatenate((embedding_matrix_glove, embedding_matrix_fasttext), axis=1)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del embedding_matrix_glove\n#del embedding_matrix_fasttext\n#gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del word_dict\ndel lemma_dict\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# hyperparameters\nmax_length = 220\nembedding_size = 300\nnum_epoch = 4\n\ntrain_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')\ntest_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')\nprint(train_word_sequences[:1])\nprint(test_word_sequences[:1])\npred_prob = np.zeros((len(test_word_sequences),), dtype=np.float32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"IDENTITY_COLUMNS = [\n    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',\n    'muslim', 'black', 'white', 'psychiatric_or_mental_illness'\n]\nAUX_COLUMNS = ['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']\nTEXT_COLUMN = 'comment_text'\nTARGET_COLUMN = 'target'\nfor column in IDENTITY_COLUMNS + [TARGET_COLUMN]:\n    train[column] = np.where(train[column] >= 0.5, True, False)\nsample_weights = np.ones(len(train), dtype=np.float32)\nsample_weights += np.array(train[IDENTITY_COLUMNS].sum(axis=1))\nsample_weights += train[TARGET_COLUMN] * (~train[IDENTITY_COLUMNS]).sum(axis=1)\nsample_weights += (~train[TARGET_COLUMN]) * train[IDENTITY_COLUMNS].sum(axis=1) * 5\nsample_weights /= sample_weights.mean()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train[TARGET_COLUMN].values\ny_aux_train = train[AUX_COLUMNS].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom keras.models import Model\nfrom keras.layers import Input, Dense, Embedding, SpatialDropout1D, add, concatenate\nfrom keras.layers import CuDNNLSTM, Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import LearningRateScheduler\n\nNUM_MODELS = 2\nLSTM_UNITS = 128\nDENSE_HIDDEN_UNITS = 4 * LSTM_UNITS\nEPOCHS = 4\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"start_time = time.time()\nprint(\"Start training ...\")\n\n\ncheckpoint_predictions = []\nweights = []\n\nfor model_idx in range(1):\n    model = build_model_simplelstm(embedding_matrix_glove, y_aux_train.shape[-1])\n    for global_epoch in range(EPOCHS):\n        model.fit(\n            train_word_sequences,\n            [y_train, y_aux_train],\n            batch_size=400,\n            epochs=1,\n            verbose=1,\n            sample_weight=[sample_weights.values, np.ones_like(sample_weights)],\n            callbacks=[\n                LearningRateScheduler(lambda _: 1e-3 * (0.55 ** global_epoch))\n            ]\n        )\n        checkpoint_predictions.append(model.predict(test_word_sequences, batch_size=2048)[0].flatten())\n        result_ = pd.DataFrame.from_dict({\n                'id': test.id,\n                'prediction': checkpoint_predictions[-1]\n                })\n        result_.to_csv(\"model_%d_%d.csv\" % (model_idx, global_epoch))\n        weights.append(2 ** global_epoch)\n   \n\nprint(\"--- %s seconds ---\" % (time.time() - start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = np.average(checkpoint_predictions, weights=weights, axis=0)\n\nsubmission = pd.DataFrame.from_dict({\n    'id': test.id,\n    'prediction': predictions\n})\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nstart_time = time.time()\nprint(\"Start training ...\")\nmodel = build_model_(embedding_matrix, nb_words, embedding_size)\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)\npred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)\npred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\ndel model, embedding_matrix_fasttext, embedding_matrix\ngc.collect()\nK.clear_session()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n\n\n\n'''\nstart_time = time.time()\nprint(\"Start training ...\")\nmodel = build_model(embedding_matrix, nb_words, embedding_size)\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)\npred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)\npred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\ndel model, embedding_matrix_fasttext, embedding_matrix\ngc.collect()\nK.clear_session()\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Loading embedding matrix ...\")\nembedding_matrix_para, nb_words = load_para(word_dict, lemma_dict)\nembedding_matrix = np.concatenate((embedding_matrix_glove, embedding_matrix_para), axis=1)\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nstart_time = time.time()\nprint(\"Start training ...\")\nmodel = build_model(embedding_matrix, nb_words, embedding_size)\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=num_epoch-1, verbose=2)\npred_prob += 0.15*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nmodel.fit(train_word_sequences, y, batch_size=batch_size, epochs=1, verbose=2)\npred_prob += 0.35*np.squeeze(model.predict(test_word_sequences, batch_size=batch_size, verbose=2))\nprint(\"--- %s seconds ---\" % (time.time() - start_time))\n\nsubmission = pd.DataFrame.from_dict({'qid': test['qid']})\nsubmission['prediction'] = (pred_prob>0.35).astype(int)\nsubmission.to_csv('submission.csv', index=False)\n'''","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}