{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from sklearn.metrics import accuracy_score\nimport numpy as np\nimport pandas as pd\nimport spacy\nfrom sklearn.ensemble import RandomForestClassifier\nimport time\nimport nltk\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import cross_val_score\nfrom nltk.stem import PorterStemmer\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import cross_validate\nimport time\nfrom scipy import sparse\nimport matplotlib.pyplot as plt\nfrom matplotlib.legend_handler import HandlerLine2D\nfrom sklearn.neural_network import MLPClassifier\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import precision_score\nfrom sklearn.metrics import recall_score\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.svm import LinearSVC\n# import xgboost as xgb\nfrom sklearn.metrics import classification_report\nfrom nltk.stem import WordNetLemmatizer\nfrom sklearn.naive_bayes import BernoulliNB\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom imblearn.over_sampling import SMOTE\nfrom sklearn.linear_model import LogisticRegression\nfrom tqdm import tqdm\nimport time\nfrom keras.preprocessing.sequence import pad_sequences\nimport os\nfrom glob import glob\nfrom zipfile import ZipFile\nimport zipfile\nfrom numpy import asarray\nfrom numpy import zeros\nfrom nltk.stem import PorterStemmer\nps = PorterStemmer()\nfrom nltk.stem.lancaster import LancasterStemmer\nlc = LancasterStemmer()\nfrom nltk.stem import SnowballStemmer\nsb = SnowballStemmer(\"english\")\nimport gc\nimport gensim\nimport io\n\ndef read_LSTM(reset_data):\n    from keras.preprocessing.text import Tokenizer\n    from keras.preprocessing import sequence\n    import os\n    max_length = 50\n    # load data\n    if(reset_data or not os.path.exists('lstm_data_vector.npy') or not os.path.exists(\"lstm_labels_vector.npy\") or not os.path.exists(\"lstm_qid_vector.npy\")):\n        start_time = time.time()\n        train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv').fillna('')\n        test = pd.read_csv('../input/quora-insincere-questions-classification/test.csv').fillna('')\n        qid = test.drop('question_text', axis=1)\n        training_labels = train['target'].to_numpy()    \n        train_text = train['question_text']\n        test_text = test['question_text']\n        text_list = pd.concat([train_text, test_text])\n        y = train['target'].values\n        num_train_data = y.shape[0]\n        start_time = time.time()\n        nlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])\n        nlp.vocab.add_flag(lambda s: s.lower() in spacy.lang.en.stop_words.STOP_WORDS, spacy.attrs.IS_STOP)\n        word_dict = {}\n        word_index = 1\n        lemma_dict = {}\n        docs = nlp.pipe(text_list, n_threads = 2)\n        word_sequences = []\n        for doc in tqdm(docs):\n            word_seq = []\n            for token in doc:\n                if (token.text not in word_dict) and (token.pos_ is not \"PUNCT\"):\n                    word_dict[token.text] = word_index\n                    word_index += 1\n                    lemma_dict[token.text] = token.lemma_\n                if token.pos_ is not \"PUNCT\":\n                    word_seq.append(word_dict[token.text])\n            word_sequences.append(word_seq)\n        del docs\n        train_word_sequences = word_sequences[:num_train_data]\n        test_word_sequences = word_sequences[num_train_data:]\n        train_word_sequences = pad_sequences(train_word_sequences, maxlen=max_length, padding='post')\n        test_word_sequences = pad_sequences(test_word_sequences, maxlen=max_length, padding='post')\n        # print(padded_train)\n\n        # Save Data\n        #np.save(\"lstm_data_vector.npy\",train_word_sequences)\n        #np.save(\"lstm_labels_vector.npy\", training_labels)\n        #np.save(\"lstm_qid_vector.npy\", qid)\n        #np.save(\"lstm_test_vector.npy\", test_word_sequences)\n        \n    else:\n        padded_train = np.load(\"lstm_data_vector.npy\")\n        training_labels = np.load(\"lstm_data_vector.npy\")\n        qid = np.load(\"lstm_qid_vector.npy\")\n        padded_test = np.load(\"lstm_test_vector.npy\")\n\n\n\n    x_train, x_test, y_train, y_test = train_test_split(train_word_sequences, training_labels, test_size=0.01, random_state=27)\n    print('x_train.shape: ', x_train.shape)\n    print('x_test.shape: ', x_test.shape)\n    print('y_train.shape: ', y_train.shape)\n    print('y_test.shape: ', y_test.shape)\n    max_question_length = 50\n    vocab_size = len(word_dict)\n    print(vocab_size)\n    return word_dict, lemma_dict, test_word_sequences, qid, x_train, x_test, y_train, y_test, vocab_size, max_question_length\n\ndef do_LSTM(word_dict, lemma_dict, actual_test, x_train, x_test, y_train, y_test, vocab_size, max_question_length):\n    from keras.models import Sequential\n    from keras.layers import Dense, LSTM\n    from keras.layers.embeddings import Embedding\n    max_length = 50\n    dim=300\n    embeddings_index={}\n    with zipfile.ZipFile(\"../input/quora-insincere-questions-classification/embeddings.zip\") as zf:\n        with io.TextIOWrapper(zf.open(\"glove.840B.300d/glove.840B.300d.txt\"), encoding=\"utf-8\") as f:\n            for line in tqdm(f):\n                values=line.split(' ')\n                word=values[0]\n                vectors=np.asarray(values[1:],'float32')\n                embeddings_index[word]=vectors\n    embeddings_index = dict(embeddings_index)\n    print('embedding stuff')\n    embed_size = 300\n    embedding_matrix = np.zeros((len(word_dict)+1, embed_size), dtype=np.float32)\n    empty = np.zeros((embed_size,), dtype=np.float32) - 1.\n    for key in tqdm(word_dict):\n        word = key\n        embedding_vector1 = embeddings_index.get(word)\n        word = key.lower()\n        embedding_vector2 = embeddings_index.get(word)\n        word = key.upper()\n        embedding_vector3 = embeddings_index.get(word)\n        word = key.capitalize()\n        embedding_vector4 = embeddings_index.get(word)\n        word = ps.stem(key)\n        embedding_vector5 = embeddings_index.get(word)\n        word = lemma_dict[key]\n        embedding_vector6 = embeddings_index.get(word)\n        if embedding_vector1 is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector1\n            continue\n        if embedding_vector2 is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector2\n            continue\n        if embedding_vector3 is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector3\n            continue\n        if embedding_vector4 is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector4\n            continue\n        if embedding_vector5 is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector5\n            continue\n        if embedding_vector6 is not None:\n            embedding_matrix[word_dict[key]] = embedding_vector6\n            continue\n        embedding_matrix[word_dict[key]] = empty        \n    \n    embedding_vecor_length = 50 # num features for each sentence?\n    model = Sequential()\n    model.add(Embedding(vocab_size+1, 300, weights=[embedding_matrix], input_length=max_length, trainable=False))\n    model.add(LSTM(100))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    print(model.summary())\n    model.fit(x_train, y_train, epochs=3, batch_size=512)\n    \n    print(\"starting predict\")\n    start = time.time()\n    predictions = model.predict_classes(x_test)\n    print('Time to predict: ', time.time() - start)    \n    \n    print(\"starting Real predict\")\n    start = time.time()\n    actual_pred = model.predict_classes(actual_test)\n    print('Time to real predict: ', time.time() - start)    \n    \n    return actual_pred, predictions\n\n\ndef main():\n    max_length = 55\n    lstm = True\n    reset_data = True\n    if lstm:\n        word_dict, lemma_dict, actual_test, qid, x_train, x_test, y_train, y_test, vocab_size, max_question_length = read_LSTM(reset_data)\n    else:\n        return\n    print(\"starting training\")\n    start = time.time()\n    # use actual_test instead of x_test to do real predictions\n    # tes = do_LSTM(x_train, x_test, y_train, y_test, vocab_size, max_question_length)\n    actual_pred, tes = do_LSTM(word_dict, lemma_dict, actual_test, x_train, x_test, y_train, y_test, vocab_size, max_question_length)\n    \n\n    print('Time to train & predict: ', time.time() - start)\n    print(\"confusion matrix:\\n\", confusion_matrix(y_test, tes))\n    print(\"accuracy: \", accuracy_score(y_test, tes))\n    print(\"f1: \", f1_score(y_test, tes, average='macro'))\n    print(\"classification report:\\n\", classification_report(y_test, tes))\n    print(\"precision: \", precision_score(y_test, tes))\n    print(\"recall: \", recall_score(y_test, tes))\n    print(actual_pred)\n    qid['prediction'] = actual_pred\n    qid.to_csv('submission.csv', index=False)\n\nif __name__ == \"__main__\":\n    main()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}