{"cells":[{"metadata":{"_cell_guid":"d3b04218-0413-4e6c-8751-5d8a404d73a9","_uuid":"0bca9739b82d5d51e1229243e03ea1b6db35c17e"},"cell_type":"markdown","source":"## Introduction\n\nThis is a replica of Jeremy Howard's kernel: https://www.kaggle.com/jhoward/nb-svm-strong-linear-baseline\nThe point of this kernel is to have a simple baseline with a linear model.\n\nThis kernel shows how to use NBSVM (Naive Bayes - Support Vector Machine) to create a baseline for the Quora Insencere Quesrtions competition. NBSVM was introduced by Sida Wang and Chris Manning in the paper [Baselines and Bigrams: Simple, Good Sentiment and Topic Classification](https://nlp.stanford.edu/pubs/sidaw12_simple_sentiment.pdf). In this kernel, we use sklearn's logistic regression, rather than SVM, although in practice the two are nearly identical (sklearn uses the liblinear library behind the scenes).\n\nIf you're not familiar with naive bayes and bag of words matrices, there's a preview available of one of fast.ai's upcoming *Practical Machine Learning* course videos, which introduces this topic. Here is a link to the section of the video which discusses this: [Naive Bayes video](https://youtu.be/37sFIak42Sc?t=3745)."},{"metadata":{"_cell_guid":"ef06cd19-66b6-46bc-bf45-184e12d3f7d4","_uuid":"cca038ca9424a3f66e10262fc9129de807b5f855","trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a494f561-0c2f-4a38-8973-6b60c22da357","_uuid":"f70ebe669fcf6b434c595cf6fb7a76120bf7809c","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\nsubm = pd.read_csv('../input/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"480780f1-00c0-4f9a-81e5-fc1932516a80","_uuid":"f2e77e8e6df5e29b620c7a2a0add1438c35af932"},"cell_type":"markdown","source":"## Building the model\n\nWe'll start by creating a *bag of words* representation, as a *term document matrix*. We'll use ngrams, as suggested in the NBSVM paper."},{"metadata":{"_cell_guid":"b7f11db7-5c12-4eb8-9f2d-0323d629fed9","_uuid":"b043a3fb66c443fab0129e863c134ec813dadb87","trusted":true},"cell_type":"code","source":"import re, string\nre_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')\ndef tokenize(s): return re_tok.sub(r' \\1 ', s).split()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bfdebf11-133c-4b12-8664-8bf64757d6cc","_uuid":"941759df15c71d42853515e4d1006f4ab000ce75"},"cell_type":"markdown","source":"Split in train and test"},{"metadata":{"trusted":true,"_uuid":"da54fc42ca482bc687d5be24f2efafd6c060f77e"},"cell_type":"code","source":"from sklearn.model_selection import GridSearchCV, train_test_split\n\ntrain_df, val_df = train_test_split(train, test_size=0.07, random_state=2018)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"450fb65a9a666428b8faf185648c34ecb02eeccd"},"cell_type":"markdown","source":"Naive Bayes SVM Classifier"},{"metadata":{"trusted":true,"_uuid":"8531830f36f6f122aa7af93d17480ed5e780a995"},"cell_type":"code","source":"from sklearn.base import BaseEstimator, ClassifierMixin\nfrom sklearn.utils.validation import check_X_y, check_is_fitted\nfrom sklearn.linear_model import LogisticRegression\nfrom scipy import sparse\nclass NbSvmClassifier(BaseEstimator, ClassifierMixin):\n    def __init__(self, C=1.0, dual=False, n_jobs=1, solver='sag'):\n        self.C = C\n        self.dual = dual\n        self.n_jobs = n_jobs\n        self.solver = solver\n        \n    def predict(self, x):\n        # Verify that model has been fit\n        check_is_fitted(self, ['_r', '_clf'])\n        return self._clf.predict(x.multiply(self._r))\n\n    def predict_proba(self, x):\n        # Verify that model has been fit\n        check_is_fitted(self, ['_r', '_clf'])\n        return self._clf.predict_proba(x.multiply(self._r))\n\n    def fit(self, x, y):\n        # Check that X and y have correct shape\n        y = y.values\n        x, y = check_X_y(x, y, accept_sparse=True)\n\n        def pr(x, y_i, y):\n            p = x[y==y_i].sum(0)\n            return (p+1) / ((y==y_i).sum()+1)\n\n        self._r = sparse.csr_matrix(np.log(pr(x,1,y) / pr(x,0,y)))\n        x_nb = x.multiply(self._r)\n        self._clf = LogisticRegression(C=self.C, dual=self.dual, n_jobs=self.n_jobs, solver=self.solver).fit(x_nb, y)\n        return self","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3793aa11edfc943f27e4d8e50b9a4d5ea58474ea"},"cell_type":"markdown","source":"This is to find optimal threshold"},{"metadata":{"trusted":true,"_uuid":"6b28cd209591e98b5570bacec6181385e174d060"},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\ndef threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fccb9b5e4b4e25e938457bbf4a13c01a290af54f"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1b78e2f5ae09e0c7515559c6a91597263dc3ec39"},"cell_type":"markdown","source":"We use trigrams up to 50000 words"},{"metadata":{"trusted":true,"_uuid":"f3e87ec3476ae196578edd15bb5389f7525a4876"},"cell_type":"code","source":"N = 50000\n\nvec = TfidfVectorizer(ngram_range=(1,3), tokenizer=tokenize,\n               min_df=3, max_df=0.9, strip_accents='unicode', use_idf=1,\n               smooth_idf=1, sublinear_tf=1, max_features=N)\n\ntrn_term_doc = vec.fit_transform(train_df['question_text'])\nval_term_doc = vec.transform(val_df['question_text'])\ntest_term_doc = vec.transform(test['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"925a6676eaec90a7064b7237e2cd5eb3dc32bd43"},"cell_type":"code","source":"model = NbSvmClassifier(dual=True, solver='liblinear', C = 1e1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f3d989e0f1745c7d41511fd5e598912209a9f21"},"cell_type":"code","source":"model.fit(trn_term_doc, train_df['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a804304e9f642f81decac724fb55d97f99ed85fc"},"cell_type":"code","source":"preds_val = model.predict_proba(val_term_doc)[:,1]\npreds_test = model.predict_proba(test_term_doc)[:,1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"00b4210d21a05ca1dcc90fe34e12d8992ef77d47"},"cell_type":"markdown","source":"Use validation set to find optimal threshold"},{"metadata":{"trusted":true,"_uuid":"41fd567019049af40b611840e3e8f29ad64a7f47"},"cell_type":"code","source":"best_threshold = threshold_search(y_true=val_df['target'], y_proba=preds_val)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"25ee1a4f16c63d2dec4774bac56e655972fe588e"},"cell_type":"markdown","source":"F1 score results"},{"metadata":{"trusted":true,"_uuid":"ab2b99dbb4a2a7948400e71e93d191f73b9b5479"},"cell_type":"code","source":"best_threshold","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fead4679f09e98a093f2eee7059ddec0405e4271"},"cell_type":"code","source":"pred_test_y = (preds_test > best_threshold['threshold']).astype(int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}