{"cells":[{"metadata":{"_uuid":"bc529cd9e2544cad69fee23dfacfe96668078e76"},"cell_type":"markdown","source":"## Quora Question Sincerity - EDA & basic predictions   \n\n*Notes:* Evaluation metric = F1 score   \nImport statements + data loading   "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt \nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport spacy\nnlp = spacy.load('en')\ntraining_data_raw = pd.read_csv(\"../input/train.csv\")\ntraining_data_raw.drop('qid', axis=1, inplace=True)\nall_labels = training_data_raw.pop('target')\nprint(f\"{len(training_data_raw):,} total training datapoints\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8734fbba1d3d78c200f84bdd8f64c810acedaaae"},"cell_type":"code","source":"print(training_data_raw.values[:2])\nprint(all_labels.value_counts())\nprint(all_labels.value_counts(True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5cc8c44651525a3da86cfd68378659e4ff072b44","scrolled":false},"cell_type":"code","source":"# Preprocessing \nimport re\nNON_CHARACTER = re.compile(r'[^A-Za-z]+') #(?u)\nNUMS = re.compile(r'\\d+')\nfrom nltk.tokenize import word_tokenize\n# from nltk.corpus import stopwords \n# STOPS = stopwords.words('english')\nfrom nltk.stem import WordNetLemmatizer\nl = WordNetLemmatizer()\ndef process(text): \n    text = text.lower().replace('\\\\', '\\\\\\\\')\n    text = NUMS.sub('XXX', text)\n    text = NON_CHARACTER.sub(' ', text)\n#     text = ' '.join([l.lemmatize(word) for word in word_tokenize(text) if word not in STOPS])\n    text = ' '.join([l.lemmatize(word) for word in word_tokenize(text)])\n    return text \n# for x in training_data_raw.values[:10]:\n#     print(x[0])\n#     print(process(x[0]))\nall_texts = np.array([process(x[0]) for x in training_data_raw.values])\nprint(\"Done!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"3dbc763f4a123450ddb0c1c467be33f04625dfce"},"cell_type":"code","source":"train_x, test_x, train_y, test_y = train_test_split(\n    all_texts, all_labels.values, test_size=0.2, random_state=0)\nprint(\"Test-train split done!\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc1feb442a5d439db3b20f2f7eda3ab4cb620d27"},"cell_type":"code","source":"vectorizer = CountVectorizer()\nvectorizer2 = CountVectorizer(min_df=0.0001, max_df=0.999, max_features=5000, ngram_range=(1,2,)) \nbow_train = vectorizer.fit_transform(train_x) \nbow_train2 = vectorizer2.fit_transform(train_x) \nprint(bow_train.shape)\nprint(bow_train2.shape)\nbow_test = vectorizer.transform(test_x)\nbow_test2 = vectorizer2.transform(test_x)\nprint(\"Done creating Bag-of-Words\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ca923cc7ceffe28a0895e9218d6609a738ac024"},"cell_type":"markdown","source":"## Model Selection \nAvailable models:   \n* [word2vec](https://code.google.com/archive/p/word2vec/) - trained by Google on GoogleNews, published 2013 \n* [GloVe](https://nlp.stanford.edu/projects/glove/) - trained by Stanford on Wikipedia?, published 2014   \n* [PARAGRAM-SL999 ](https://cogcomp.org/page/resource_view/106) - initialized by GloVE, trained by UIUC+TTIC on the Paraphrase Database, published 2015\n* [fasttext](https://fasttext.cc/docs/en/english-vectors.html) - trained by FastText (Facebook AI) on Gigaword+Wikipedia+Common Crawl+stamt.org news+UMBC news, published 2017   \n\nPerformance (as evaulated by MEN [here](https://github.com/kudkudak/word-embeddings-benchmarks/wiki) [and [here](https://arxiv.org/pdf/1805.07966.pdf)]): fasttext=0.805, word2vec=0.741[0.78], GloVe=0.7365[0.80], SL999=[0.78]\n"},{"metadata":{"trusted":true,"_uuid":"b296a37273b010b0ac95054f8de2bfd47ba3a8e6"},"cell_type":"code","source":"from sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3ade60ce4a86f1562caaac9c5b6d6badc889091"},"cell_type":"markdown","source":"### Logistic Regression"},{"metadata":{"trusted":true,"_uuid":"1e09829fc2898325dbc7580905775d41c5bb719f","scrolled":true},"cell_type":"code","source":"print(f\"Results of logistic regression on full bag-of-words\")\nlogistic = LogisticRegression(penalty=\"l1\", C=3.5) \nlogistic.fit(bow_train, train_y) \ntrain_predictions = logistic.predict(bow_train)\ntrain_acc = accuracy_score(train_y, train_predictions)  #all_labels\ntrain_f1 = f1_score(train_y, train_predictions) \nprint(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \ntest_predictions = logistic.predict(bow_test)\ntest_acc = accuracy_score(test_y, test_predictions) \ntest_f1 = f1_score(test_y, test_predictions) \nprint(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")\n# Training accuracy: 96.37%, F1: 0.6580, %1: 4.42%\n# Testing accuracy:  95.33%, F1: 0.5504, %1: 4.22%","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02d88d9da580a5d8ac3c8168028eead9f2663b2a"},"cell_type":"code","source":"print(f\"Results of logistic regression on simplified bag-of-words\")\nlogistic2 = LogisticRegression(penalty=\"l1\", C=3.5) \nlogistic2.fit(bow_train2, train_y)\ntrain_predictions = logistic2.predict(bow_train2)\ntrain_acc = accuracy_score(train_y, train_predictions) \ntrain_f1 = f1_score(train_y, train_predictions) \nprint(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \ntest_predictions = logistic2.predict(bow_test2)\ntest_acc = accuracy_score(test_y, test_predictions) \ntest_f1 = f1_score(test_y, test_predictions) \nprint(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")\n# Training accuracy: 95.26%, F1: 0.5187, %1: 3.65%\n# Testing accuracy:  95.14%, F1: 0.5051, %1: 3.66%","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cd0789a3d8d04827e4af7187ebf545f24e3771b8"},"cell_type":"markdown","source":"### Decision Tree"},{"metadata":{"trusted":true,"_uuid":"c9f60a96042f4e5fefdf3450d9ff2828883f6f6d"},"cell_type":"code","source":"# print(f\"Results of decision tree on full bag-of-words\")\n# dtc = DecisionTreeClassifier(max_depth=30) \n# dtc.fit(bow_train, train_y)\n# train_predictions = dtc.predict(bow_train)\n# train_acc = accuracy_score(train_y, train_predictions) \n# train_f1 = f1_score(train_y, train_predictions) \n# print(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \n# test_predictions = dtc.predict(bow_test)\n# test_acc = accuracy_score(test_y, test_predictions) \n# test_f1 = f1_score(test_y, test_predictions) \n# print(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"29e1b38aaf28424e9757c645efa43b1d0fdf4ebd"},"cell_type":"code","source":"# print(f\"Results of decision tree on simplified bag-of-words\")\n# dtc2 = DecisionTreeClassifier(max_depth=30) \n# dtc2.fit(bow_train2, train_y)\n# train_predictions = dtc2.predict(bow_train2)\n# train_acc = accuracy_score(train_y, train_predictions) \n# train_f1 = f1_score(train_y, train_predictions) \n# print(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \n# test_predictions = dtc2.predict(bow_test2)\n# test_acc = accuracy_score(test_y, test_predictions) \n# test_f1 = f1_score(test_y, test_predictions) \n# print(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"87d16242b558310482b20779fc71530e0c9f149b"},"cell_type":"markdown","source":"### Random Forest"},{"metadata":{"trusted":true,"_uuid":"ef04d8983429f5a5c76efe4916582cb9c64e594b"},"cell_type":"code","source":"# print(f\"Results of random forest on full bag-of-words\")\n# rfc = RandomForestClassifier(n_estimators=100, max_depth=3, class_weight='balanced') \n# rfc.fit(bow_train, train_y)\n# train_predictions = rfc.predict(bow_train)\n# train_acc = accuracy_score(train_y, train_predictions) \n# train_f1 = f1_score(train_y, train_predictions) \n# print(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \n# test_predictions = rfc.predict(bow_test)\n# test_acc = accuracy_score(test_y, test_predictions) \n# test_f1 = f1_score(test_y, test_predictions) \n# print(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c15a50febd6f6e96ecf3591f4b50bad8d4ed0f11"},"cell_type":"code","source":"# print(f\"Results of random forest on simplified bag-of-words\") \n# rfc2 = RandomForestClassifier(n_estimators=100, max_depth=3, class_weight='balanced') \n# rfc2.fit(bow_train2, train_y) \n# train_predictions = rfc2.predict(bow_train2)\n# train_acc = accuracy_score(train_y, train_predictions) \n# train_f1 = f1_score(train_y, train_predictions) \n# print(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \n# test_predictions = rfc2.predict(bow_test2)\n# test_acc = accuracy_score(test_y, test_predictions) \n# test_f1 = f1_score(test_y, test_predictions) \n# print(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3ade60ce4a86f1562caaac9c5b6d6badc889091"},"cell_type":"markdown","source":"### Gradient Boosting"},{"metadata":{"trusted":true,"_uuid":"1e09829fc2898325dbc7580905775d41c5bb719f","scrolled":true},"cell_type":"code","source":"# print(f\"Results of gradient boosting on full bag-of-words\")\n# gbc = GradientBoostingClassifier() \n# gbc.fit(bow_train, train_y) \n# train_predictions = gbc.predict(bow_train)\n# train_acc = accuracy_score(train_y, train_predictions) \n# train_f1 = f1_score(train_y, train_predictions) \n# print(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \n# test_predictions = gbc.predict(bow_test)\n# test_acc = accuracy_score(test_y, test_predictions) \n# test_f1 = f1_score(test_y, test_predictions) \n# print(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")\n# # Training accuracy: 94.56%, F1: 0.3035, %1: 1.61%\n# # Testing accuracy:  94.58%, F1: 0.3014, %1: 1.61%","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02d88d9da580a5d8ac3c8168028eead9f2663b2a"},"cell_type":"code","source":"# print(f\"Results of gradient boosting on simplified bag-of-words\")\n# gbc2 = GradientBoostingClassifier() \n# gbc2.fit(bow_train2, train_y)\n# train_predictions = gbc2.predict(bow_train2)\n# train_acc = accuracy_score(train_y, train_predictions) \n# train_f1 = f1_score(train_y, train_predictions) \n# print(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \n# test_predictions = gbc2.predict(bow_test2)\n# test_acc = accuracy_score(test_y, test_predictions) \n# test_f1 = f1_score(test_y, test_predictions) \n# print(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")\n# # Training accuracy: 94.59%, F1: 0.3123, %1: 1.67%\n# # Testing accuracy:  94.60%, F1: 0.3095, %1: 1.66%","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41482f66ace07ac370c24d352cfc1d38d646eef8"},"cell_type":"markdown","source":"## Final Model & Submission "},{"metadata":{"trusted":true,"_uuid":"8d9bfb6d4588f97cae591e2d5ff118049e5b6249"},"cell_type":"code","source":"validation_data = pd.read_csv(\"../input/test.csv\")\nfinal_vectorizer = CountVectorizer()\n# final_vectorizer = CountVectorizer(min_df=0.0005, max_features=5000, ngram_range=(1,2,)) \nfinal_model = LogisticRegression(penalty=\"l1\", C=3.5)\n# final_model = RandomForestClassifier(n_estimators=100, max_depth=3, class_weight='balanced') \n### Code running \nprint(f\"Generating final model\")\nfinal_bow = final_vectorizer.fit_transform(all_texts) \nfinal_model.fit(final_bow, all_labels.values) \nfinal_train_predictions = final_model.predict(final_bow) \nfinal_acc = accuracy_score(all_labels.values, final_train_predictions) \nfinal_f1 = f1_score(all_labels.values, final_train_predictions) \nprint(f\"Final model accuracy:  {final_acc:.2%}, F1: {final_f1:.4f}, %1: {sum(final_train_predictions)/len(final_train_predictions):.2%}, %1 actual: {sum(all_labels.values)/len(all_labels.values):.2%}\") \n# LogisticRegression(penalty=\"l1\") Final model accuracy:  95.73%, F1: 0.5814, %1: 4.02%, %1 actual: 6.19%\n# LogisticRegression(penalty=\"l1\", C=2.5) Final model accuracy:  96.23%, F1: 0.6415, %1: 4.32%, %1 actual: 6.19%\n# LogisticRegression(penalty=\"l1\", C=3.0) Final model accuracy:  96.27%, F1: 0.6466, %1: 4.37%, %1 actual: 6.19%\n# LogisticRegression(penalty=\"l1\", C=3.5)\nvalidation_texts = np.array([process(x) for x in validation_data['question_text']])\nprint(validation_texts[:3])\nvalidation_bow = final_vectorizer.transform(validation_texts) \nvalidation_predictions = final_model.predict(validation_bow) \nprint(validation_predictions[:3])\nprint(f\"Submission %1: {sum(validation_predictions)/len(validation_predictions):.2%}\")\nvalidation_data.drop('question_text', axis=1, inplace=True)\nvalidation_data['prediction'] = validation_predictions \nvalidation_data.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}