{"cells":[{"metadata":{"_uuid":"2545a3ad7f22c0d72d465b639be421320a7d0896"},"cell_type":"markdown","source":"# Quora Insincere Questions Sklearn Baseline Models with Downsampling\nIn this kernel we will try several of sklearn's classification algorithms (plus XGBoost) to find a good baseline.  We will then select the best performing models and try to improve performance by adjusting hyperparameters and features."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import packages\nimport numpy as np\nimport pandas as pd\n\nimport spacy\nimport re\n\nfrom gensim import corpora, models, similarities\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\n\nnp.random.seed(27)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eaddf3f534ef2e9fe14142e844ce12426453961b"},"cell_type":"markdown","source":"#### Text Pre-processing and Downsampling\nBecause we have an imbalanced dataset we will downsample the majority class to equal the size of the minority class.  This will not only balance our dataset, but will decrease processing time due to the decreased number of samples in the training data."},{"metadata":{"trusted":true,"_uuid":"280ef1c0380632da7c0867e1887195e026e2afd3"},"cell_type":"code","source":"# taking a small sample (with downsampling of majority class) of the training data to speed up processing\nfrom sklearn.utils import resample\n\nsincere = train[train.target == 0]\ninsincere = train[train.target == 1]\n\ntrain = pd.concat([resample(sincere,\n                     replace = False,\n                     n_samples = len(insincere)), insincere])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee99bc21de267796eccacc8941a35d4e1667f6de"},"cell_type":"code","source":"contractions = {\n\"ain't\": \"is not\",\n\"aren't\": \"are not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he will\",\n\"he'll've\": \"he he will have\",\n\"he's\": \"he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how is\",\n\"I'd\": \"I would\",\n\"I'd've\": \"I would have\",\n\"I'll\": \"I will\",\n\"I'll've\": \"I will have\",\n\"I'm\": \"I am\",\n\"I've\": \"I have\",\n\"i'd\": \"i would\",\n\"i'd've\": \"i would have\",\n\"i'll\": \"i will\",\n\"i'll've\": \"i will have\",\n\"i'm\": \"i am\",\n\"i've\": \"i have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it will\",\n\"it'll've\": \"it will have\",\n\"it's\": \"it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she will\",\n\"she'll've\": \"she will have\",\n\"she's\": \"she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so as\",\n\"that'd\": \"that would\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that is\",\n\"there'd\": \"there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there is\",\n\"they'd\": \"they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they will\",\n\"they'll've\": \"they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\"we'd\": \"we would\",\n\"we'd've\": \"we would have\",\n\"we'll\": \"we will\",\n\"we'll've\": \"we will have\",\n\"we're\": \"we are\",\n\"we've\": \"we have\",\n\"weren't\": \"were not\",\n\"what'll\": \"what will\",\n\"what'll've\": \"what will have\",\n\"what're\": \"what are\",\n\"what's\": \"what is\",\n\"what've\": \"what have\",\n\"when's\": \"when is\",\n\"when've\": \"when have\",\n\"where'd\": \"where did\",\n\"where's\": \"where is\",\n\"where've\": \"where have\",\n\"who'll\": \"who will\",\n\"who'll've\": \"who will have\",\n\"who's\": \"who is\",\n\"who've\": \"who have\",\n\"why's\": \"why is\",\n\"why've\": \"why have\",\n\"will've\": \"will have\",\n\"won't\": \"will not\",\n\"won't've\": \"will not have\",\n\"would've\": \"would have\",\n\"wouldn't\": \"would not\",\n\"wouldn't've\": \"would not have\",\n\"y'all\": \"you all\",\n\"y'all'd\": \"you all would\",\n\"y'all'd've\": \"you all would have\",\n\"y'all're\": \"you all are\",\n\"y'all've\": \"you all have\",\n\"you'd\": \"you would\",\n\"you'd've\": \"you would have\",\n\"you'll\": \"you will\",\n\"you'll've\": \"you will have\",\n\"you're\": \"you are\",\n\"you've\": \"you have\"\n}\n\nc_re = re.compile('(%s)' % '|'.join(contractions.keys()))\n\ndef expandContractions(text, c_re=c_re):\n    def replace(match):\n        return contractions[match.group(0)]\n    return c_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2dd1693f3eccef038cff273125e934b55af1003"},"cell_type":"code","source":"# function to clean and lemmatize text and remove stopwords\nfrom gensim.parsing.preprocessing import preprocess_string\nfrom gensim.parsing.preprocessing import strip_tags, strip_punctuation, strip_numeric\nfrom gensim.parsing.preprocessing import strip_multiple_whitespaces, strip_non_alphanum, remove_stopwords, strip_short\n\nCUSTOM_FILTERS = [lambda x: x.lower(), #lowercase\n                  strip_tags, # remove html tags\n                  strip_punctuation, # replace punctuation with space\n                  strip_multiple_whitespaces,# remove repeating whitespaces\n                  strip_non_alphanum, # remove non-alphanumeric characters\n                  strip_numeric, # remove numbers\n                  remove_stopwords,# remove stopwords\n                  strip_short # remove words less than minsize=3 characters long\n                 ]\nnlp = spacy.load('en')\n\ndef gensim_preprocess(docs, logging=True):\n    docs = [expandContractions(doc) for doc in docs]\n    docs = [preprocess_string(text, CUSTOM_FILTERS) for text in docs]\n    texts_out = []\n    for doc in docs:\n    # https://spacy.io/usage/processing-pipelines\n        doc = nlp((\" \".join(doc)),  # doc = text to tokenize => creates doc\n                  # disable parts of the language processing pipeline we don't need here to speed up processing\n                  disable=['ner', # named entity recognition\n                           'tagger', # part-of-speech tagger\n                           'textcat', # document label categorizer\n                          ])\n        texts_out.append([tok.lemma_ for tok in doc if tok.lemma_ != '-PRON-'])\n    return pd.Series(texts_out)\n\ngensim_preprocess(train.question_text.iloc[10:15])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cbef750800237534670c371bcf4103d18e06529d"},"cell_type":"code","source":"# apply text-preprocessing function to training set\n%time train_corpus = gensim_preprocess(train.question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ce2d567ac6c5acd054db57b1668afed85238c9f"},"cell_type":"code","source":"# create ngrams\nngram_phraser = models.Phrases(train_corpus, threshold=1)\nngram = models.phrases.Phraser(ngram_phraser)\n#print example\nprint(ngram[train_corpus[0]])\n\n# apply model to corpus\ntexts = [ngram[token] for token in train_corpus]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"583d3666f78e0c4a4c881020462a77fdb75e2a64"},"cell_type":"code","source":"# preparing ngrams for modeling\ntexts = [' '.join(text) for text in texts]\ntrain['ngrams'] = texts\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d1463ba8f2e18e348ab6102dd2cf91c5cf16c815"},"cell_type":"code","source":"# represent features as BOW\nfrom sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer = CountVectorizer()\nvectorizer.fit(train.ngrams)\n\n# split into test and train sets\nX_train, X_test, y_train, y_test = train_test_split(train.ngrams, train.target, test_size=0.2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7bc9ba1b315c6b1b212483f6a1d34ffe2f406d7a"},"cell_type":"markdown","source":"## Logistic Regression Baseline Model\nLogistic Regression is a linear model for classification.  They are fast to train and predict, scale well, and are easy to interpret, and are therefore a good choice for a baseline model."},{"metadata":{"trusted":true,"_uuid":"e0df2aecb50730e61c034849bdcf2f7e4859e379"},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\n\nlr = LogisticRegression()\nlr.fit(vectorizer.transform(X_train), y_train)\n\nprint('Logistic Regression Score: ', lr.score(vectorizer.transform(X_test), y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"48b67400149772936e26fcfc948c31e34f3ca7c4"},"cell_type":"code","source":"y_ = lr.predict(vectorizer.transform(X_test))\n\nfrom sklearn.metrics import classification_report\nprint(classification_report(y_test, y_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2178c186bf7d20e290985e7a5ee38101506b6926"},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix\npd.DataFrame(confusion_matrix(y_test, y_))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0c4438615a0960c68eb49e0c68d386b2d65a2706"},"cell_type":"markdown","source":"## Interpreting the Results - Classification Report\n\n* **Precision** is the number of true positives divided by all positive predictions. Precision is also called Positive Predictive Value. It is a measure of a classifier's exactness. Low precision indicates a high number of false positives.\n* **Recall** is the number of true positives divided by the number of positive values in the test data. Recall is also called Sensitivity or the True Positive Rate. It is a measure of a classifier's completeness. Low recall indicates a high number of false negatives.\n* **F1-Score** is the harmonic mean of precision and recall.\n* **Support** is the number of true results for each class."},{"metadata":{"_uuid":"29387b5be1a7b43c871c68c1cb1c9889ed3ed302"},"cell_type":"markdown","source":"## Bernoulli Naive Bayes Baseline Model\nNaive Bayes classifiers are super fast to train and work well with high-dimensional sparse data, including text. They are based on applying [Bayes' Theorem](https://en.wikipedia.org/wiki/Bayes%27_theorem) and are 'naive' in that they assume independence between features.  Scikit-Learn implements several types of Naive Bayes classifiers that are widely used for text data including Bernoulli (which we use here) and Multinomial."},{"metadata":{"trusted":true,"_uuid":"5639d7f1ecc2e7cab4a25f41774b5ec42f72ee61"},"cell_type":"code","source":"from sklearn.naive_bayes import BernoulliNB\n\nbnb = BernoulliNB()\n%time bnb.fit(vectorizer.transform(X_train), y_train)\n\nprint('Naive Bayes Score: ', bnb.score(vectorizer.transform(X_test), y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"68fe367dc934fa9b9a15bc347b13e21f2cf26dd0"},"cell_type":"code","source":"%time bnb_y_ = bnb.predict(vectorizer.transform(X_test))\n\nprint(classification_report(y_test, bnb_y_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96873658a757d0103f49d904a23bbe30995f9c0a"},"cell_type":"code","source":"pd.DataFrame(confusion_matrix(y_test, bnb_y_))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5a4bacb624a705b957052266bcde6aa57b25db9d"},"cell_type":"markdown","source":"## XGBoost Model\nXGBoost (Extreme Gradient Boosting) is an implementation of gradient boosted decision trees designed for speed and performance.  As such, it often outperforms other algorithms."},{"metadata":{"trusted":true,"_uuid":"38cd218ae07650611ab41b8d100491efa7ce04e5"},"cell_type":"code","source":"import xgboost as xgb\n\nxgb_model = xgb.XGBClassifier().fit(vectorizer.transform(X_train), y_train)\n\nprint('XGBoost Score: ', xgb_model.score(vectorizer.transform(X_test), y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d20cd99dde6f695ff51d9749d08f18c8b2ecec2e"},"cell_type":"code","source":"xgb_y_ = xgb_model.predict(vectorizer.transform(X_test))\n\nprint(classification_report(y_test, xgb_y_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2ff13fb4e91f6ac63fc226b4b08dba245f9fcd5a"},"cell_type":"code","source":"pd.DataFrame(confusion_matrix(y_test, xgb_y_))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f22442bb2605c6920c110c274363b10be332f2e8"},"cell_type":"markdown","source":"## Ensemble Model\n[Scikit-learn](https://scikit-learn.org/stable/modules/ensemble.html#ensemble) states that \"the goal of ensemble methods is to combine the predictions of several base estimators built with a given learning algorithm in order to improve generalizability / robustness over a single estimator.\"  We will combine our above three models using scikit-learn's voting classifier."},{"metadata":{"trusted":true,"_uuid":"034de9e9ca46b742b13f370384817cf688818274"},"cell_type":"code","source":"from sklearn.ensemble import VotingClassifier\n\n#create submodels\nestimators = []\n\nmodel1 = lr\nmodel2 = bnb\nmodel3 = xgb_model\n\n\nestimators.append(('logistic', model1))\nestimators.append(('bernoulli', model2))\nestimators.append(('xgboost', model3))\n\n\n# create ensemble model\n%time ensemble = VotingClassifier(estimators).fit(vectorizer.transform(X_train), y_train)\nprint('Ensemble Score: ', ensemble.score(vectorizer.transform(X_test), y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"072af2038bbc63996e6357752c191101453d2018"},"cell_type":"code","source":"ensemble_y_ = ensemble.predict(vectorizer.transform(X_test))\n\nprint(classification_report(y_test, ensemble_y_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b6b2eb71b612fe8eefbf0acf964b2328a01a778"},"cell_type":"code","source":"pd.DataFrame(confusion_matrix(y_test, ensemble_y_))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eaa57ce931a13a8dcf16e193f7aa59e2650c4c20"},"cell_type":"markdown","source":"## Interpret Results\nLogistic Regression F1: 86.9  \nNaive Bayes F1: 86.5  \nXGBoost F1: 70.9  \nEnsemble F1: 86.6  \n\n## Submit to Competition"},{"metadata":{"trusted":true,"_uuid":"9e832b04c742c61fc183488ff79297235a63cdf0"},"cell_type":"code","source":"# preprocessing/lemmatizing/stemming test data\n%time test_corpus = gensim_preprocess(test.question_text)\ntest_texts = [ngram[token] for token in test_corpus]\n\ntest_texts = [' '.join(text) for text in test_texts]\ntest['ngrams'] = test_texts\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1f4ed4fbc0c4e60d8e033ae27f59ad29b587796"},"cell_type":"code","source":"#ensemble on test data\nensemble.fit(vectorizer.transform(train.ngrams), train.target)\nprediction = ensemble.predict(vectorizer.transform(test.ngrams))\n\nsubmission = pd.DataFrame({'qid':test.qid, 'prediction':prediction})\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}