{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom collections import defaultdict, Counter\nimport nltk\nfrom nltk.corpus import stopwords, brown\nfrom nltk import word_tokenize\nfrom nltk.util import ngrams\nimport math\nstop_words = set(stopwords.words('english'))\nimport warnings\nwarnings.filterwarnings('ignore')\nimport matplotlib.pyplot as plt\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"#importing the data\ntrain_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \", train_df.shape)\nprint(\"Test shape : \", test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"044b9517c38fde5a09869eb42779ec3739509cae"},"cell_type":"markdown","source":"#### Baseline Model"},{"metadata":{"trusted":true,"_uuid":"988d47dd82b20cce5d3e2e5b2bc176946cfb34c7"},"cell_type":"code","source":"#nlp/machine learning libraries\nfrom sklearn.pipeline import Pipeline, FeatureUnion\nfrom sklearn.base import BaseEstimator, TransformerMixin\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.preprocessing import StandardScaler\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.naive_bayes import MultinomialNB\nfrom xgboost import XGBClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.preprocessing import FunctionTransformer\nfrom sklearn import model_selection, preprocessing, linear_model, naive_bayes, metrics, svm\nfrom sklearn.model_selection import train_test_split,StratifiedShuffleSplit,GridSearchCV\nfrom sklearn.metrics import f1_score,classification_report,roc_curve,precision_recall_curve,auc,average_precision_score\nfrom sklearn.feature_selection import chi2, SelectKBest\nimport re\nimport pandas, xgboost, numpy, textblob, string\nimport lime\nfrom lime import lime_text\nfrom lime.lime_text import LimeTextExplainer\nimport gensim\nfrom gensim.models import Doc2Vec\nfrom gensim.models.doc2vec import TaggedDocument","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"833db3590be1011b4d1b3ac2a58106c1acf06813"},"cell_type":"code","source":"#display top 5 rows\ntrain_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b5698855cba03bbb3173dd400e892881d7cbc05"},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"01b61115af5dce952d6a5512040baa5ff67f8edb"},"cell_type":"markdown","source":"### Using only text column (question_text) for building models"},{"metadata":{"trusted":true,"_uuid":"d1bfbd82e5a0b821273066d5ce82d35b6e9dd248"},"cell_type":"code","source":"#features\nX = train_df['question_text']\n#target label\nY = train_df['target']\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3b7d3bd2ac32ba87152a3c3ab4b9fbf775152580"},"cell_type":"markdown","source":"## Baseline model - Logistic Regression"},{"metadata":{"trusted":true,"_uuid":"e31caf13bdedfb2ab95e0343f5d010255345c660"},"cell_type":"code","source":"#pipeline for creating tf idf and  basic logistic regression model\nbaseline_ngram_lr = Pipeline([\n                    ('tfidf', TfidfVectorizer(stop_words=stop_words,ngram_range=(1,3))),\n                    ('classifier', LogisticRegression()),\n                    ])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ec9150c8dac107b1cd265c1949bd66f23e592eb"},"cell_type":"code","source":"#fitting the pipeline to the train data\nbaseline_ngram_lr.fit(X_train, y_train )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd4ae0d649a11c7b11eb6efd7ffa3dca75fd29f0"},"cell_type":"code","source":"baseline_ngram_lr_preds = baseline_ngram_lr.predict(X_test)\nprint(classification_report(y_test, baseline_ngram_lr_preds))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"43f0df305ade92902f819081943db72358fbbf62"},"cell_type":"code","source":"baseline_ngram_lr_preds_prob = baseline_ngram_lr.predict_proba(X_test)[:,1]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5fde54c9e27fe9118f558ccddbb23c8d9eaf565b"},"cell_type":"markdown","source":"#### Choosing Optimal threshold with better F1 score"},{"metadata":{"trusted":true,"_uuid":"24d6deec88d9523d06431040910c99c53afb73a3"},"cell_type":"code","source":"f1_list = []\nfor threshold in np.arange(0.1, 0.6, 0.01):\n    threshold = np.round(threshold, 2)\n    f1_list.append((f1_score(y_test, (baseline_ngram_lr_preds_prob>threshold).astype(int)),threshold))\n    print(\"F1 score at threshold {0} is {1}\".format(threshold, f1_score(y_test, (baseline_ngram_lr_preds_prob>threshold).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93e4d326b59571759885c9c277a081badb18ba92","trusted":true},"cell_type":"code","source":"def sort_tuple(tup):\n    return tup[0]\n\nbest_threshold = sorted(f1_list,key=sort_tuple, reverse=True)[0][1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fdc2ccecccc525c23f79a3125b4e69ef5ce6c08f"},"cell_type":"code","source":"##creating a submission file with the optimal threshold with the baseline model\ndef submission(df, predictions, file_name, threshold=0.20):\n    print('Optimal threshold with better F1 score is: ', threshold)\n    results = (predictions > threshold).astype(int)\n    df['prediction'] = results\n    file = (file_name + '.csv')\n    df.to_csv(file, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c79ba0dfb1d288b7da18b536a5fc1d8beab43e57"},"cell_type":"code","source":"#predicting the classes on test data\nbaseline_ngram_lr_preds_prob = baseline_ngram_lr.predict_proba(test_df['question_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21f141679a3458bf952a141f8688f595f0913b1e"},"cell_type":"code","source":"print('Saving the results in the submission file')\nsub_df = pd.read_csv('../input/sample_submission.csv')\nsubmission(sub_df, baseline_ngram_lr_preds_prob, 'submission', threshold=best_threshold)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23429f5fd279787e7fa2ad865b93eaab7710fe6c"},"cell_type":"code","source":"print(\"At threshold {0}, we are getting better F1 score and we will be choosing this threshold for our submission. This is our baseline and we will try to beat this score\".format(best_threshold))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"737552d1623d87c6aa805d12caa6a4353060f48b"},"cell_type":"markdown","source":"### Model Interpretability using LIME"},{"metadata":{"trusted":true,"_uuid":"b1a9c4907410a3c218cd9203b769c872aa1c2f60"},"cell_type":"code","source":"explainer = LimeTextExplainer(class_names=['sincere','insincere'])\nidx = 117\nexp = explainer.explain_instance(test_df['question_text'][idx], baseline_ngram_lr.predict_proba, num_features=5)\nexp.show_in_notebook(text=test_df['question_text'][idx])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26308b1a926924bd8bf401ccd4eea12aa84b012b"},"cell_type":"code","source":"idx =  56368\nexp = explainer.explain_instance(test_df['question_text'][idx], baseline_ngram_lr.predict_proba, num_features=5)\nexp.show_in_notebook(text=test_df['question_text'][idx])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24bf77141efb32449ba4e9e4241ac1e7bbb87b5d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}