{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"# all import statements\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn.metrics import accuracy_score\n#from wordcloud import WordCloud as wc   # not needed\nfrom nltk.corpus import stopwords\nimport matplotlib.pylab as pylab\nimport matplotlib.pyplot as plt\nfrom pandas import get_dummies\nimport matplotlib as mpl\nimport seaborn as sns\nimport pandas as pd\nimport numpy as np\nimport matplotlib\nimport warnings\nimport sklearn\nimport string\nimport scipy\nimport numpy\nimport nltk\nimport json\nimport sys\nimport csv\nimport os","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2  We would first do EDA ( Exploratory Data Analysis ) over Quora Data set :\n"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"print(train.info())\nprint(test.info())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# shape for train and test\nprint('Shape of train:',train.shape)\nprint('Shape of test:',test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data Cleaning\n\nWhen dealing with real-world data, dirty data is the norm rather than the exception.\n\nThe primary goal of data cleaning is to detect and remove errors and anomalies to increase the value of data in analytics and decision making. While it has been the focus of many researchers for several years, individual problems have been addressed separately. These include missing value imputation, outliers detection, transformations, integrity constraints violations detection and repair, consistent query answering, deduplication, and many other related problems such as profiling and constraints mining"},{"metadata":{"trusted":true},"cell_type":"code","source":"# How many NA elements in every column!!\n# Good news, it is Zero!\n# To check out how many null info are on the dataset, we can use isnull().sum().\n# recall from info() -> we found that it has zero Nulls. \n\ntrain.isnull().sum()\n\n# data is infact clean and ready for use.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# in case , their were NA or None values in any row then we would drop the row.\n\n# remove rows that have NA's\nprint('Before Droping',train.shape)\ntrain = train.dropna()\nprint('After Droping',train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of words in the text\n\ntrain[\"num_words\"] = train[\"question_text\"].apply(lambda x: len(str(x).split()))\ntest[\"num_words\"] = test[\"question_text\"].apply(lambda x: len(str(x).split()))\nprint('maximum of num_words in train',train[\"num_words\"].max())\nprint('min of num_words in train',train[\"num_words\"].min())\nprint(\"maximum of  num_words in test\",test[\"num_words\"].max())\nprint('min of num_words in train',test[\"num_words\"].min())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of unique words in the text\ntrain[\"num_unique_words\"] = train[\"question_text\"].apply(lambda x: len(set(str(x).split())))\ntest[\"num_unique_words\"] = test[\"question_text\"].apply(lambda x: len(set(str(x).split())))\n\nprint('maximum of num_unique_words in train',train[\"num_unique_words\"].max())\n\nprint(\"maximum of num_unique_words in test\",test[\"num_unique_words\"].max())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of stopwords in the text\n\n#from nltk.corpus import stopwords\neng_stopwords = set(stopwords.words(\"english\"))\n\ntrain[\"num_stopwords\"] = train[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\ntest[\"num_stopwords\"] = test[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in eng_stopwords]))\n\nprint('maximum of num_stopwords in train',train[\"num_stopwords\"].max())\nprint(\"maximum of num_stopwords in test\",test[\"num_stopwords\"].max())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of punctuations in the text\n\ntrain[\"num_punctuations\"] =train['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\ntest[\"num_punctuations\"] =test['question_text'].apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\nprint('maximum of num_punctuations in train',train[\"num_punctuations\"].max())\nprint(\"maximum of num_punctuations in test\",test[\"num_punctuations\"].max())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# lets figure out how many unique target values exist.\n# like we expect : 0 -> sincere qns and 1 -> un-sincere qns\n\n# You see number of unique item for Target with command below:\ntrain_target = train['target'].values\n\nnp.unique(train_target)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"YES, quora problem is a binary classification!\n\nThe data is absolutely clean. But is it balanced ?\n\nI mean do we have equal no. of sincere and un-sincere questions in the dataset.\n\nlets find out :"},{"metadata":{"trusted":true},"cell_type":"code","source":"#train.where(train ['target']==1).count()\ntrain[train.target==1].count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Such a dataset is imbalanced.\n\nImbalanced dataset is relevant primarily in the context of supervised machine learning involving two or more classes.\n\nImbalance means that the number of data points available for different classes is different: If there are two classes, then balanced data would mean 50% points for each of the class. For most machine learning techniques, little imbalance is not a problem. So, if there are 60% points for one class and 40% for the other class, it should not cause any significant performance degradation. Only when the class imbalance is high, e.g. 90% points for one class and 10% for the other, standard optimization criteria or performance measures may not be as effective and would need modification.\n\nimbalacedQuoraData\n\nA typical example of imbalanced data is encountered in e-mail classification problem where emails are classified into ham or spam. The number of spam emails is usually lower than the number of relevant (ham) emails. So, using the original distribution of two classes leads to imbalanced dataset.\n\nUsing accuracy as a performace measure for highly imbalanced datasets is not a good idea. For example, if 90% points belong to the true class in a binary classification problem, a default prediction of true for all data points leads to a classifier which is 90% accurate, even though the classifier has not learnt anything about the classification problem at hand!"},{"metadata":{"trusted":true},"cell_type":"code","source":"# visualising the imbalance in data set\n\nax=sns.countplot(x='target',hue=\"target\", data=train  ,linewidth=5,edgecolor=sns.color_palette(\"dark\", 3))\nplt.title('Is data set imbalance?');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 3  Data Preprocessing"},{"metadata":{},"cell_type":"markdown","source":"3  Data Preprocessing\nThis basically involves transforming raw data into an understandable format for NLP models.\n\nRemember, our feature \"Question_Text\" is Text or String Object and No ML algo say KNN or Bayes classification would accept Text. Hence Pre-Processing is mandatory in this case.\n\nBelow, I have recalled the two most important techniques that are also performed besides other easy to understand steps in data pre-processing:\n\nTokenization: This is a process of breaking a stream of text up into words, phrases, symbols, or other meaningful elements called tokens. The list of tokens becomes input for further processing. NLTK Library has word_tokenize and sent_tokenize to easily break a stream of text into a list of words or sentences, respectively.\nWord Stemming/Lemmatization: The aim of both processes is the same, reducing the inflectional forms of each word into a common base or root. Lemmatization is closely related to stemming. The difference is that a stemmer operates on a single word without knowledge of the context, and therefore cannot discriminate between words which have different meanings depending on part of speech. However, stemmers are typically easier to implement and run faster, and the reduced accuracy may not matter for some applications.\nQuora_data_preprocessing\n\nWe would atleast the following (Text) Pre-Processing steps\n\n1.Change all the text to lower case\n\n2.Word Tokenization\n\n3.Remove Stop words\n\n4.Remove Non-alpha text\n\n5.Word Lemmatization\n\n5.Converting the text data into Numeric vectors( called Vectorization )"},{"metadata":{"trusted":true},"cell_type":"code","source":"# step 1: Change all the text to lower case. \n\n# This is required as python interprets 'quora' and 'QUORA' differently\n\ntrain['question_text'] = [entry.lower() for entry in train['question_text']]\n\ntest['question_text'] = [entry.lower() for entry in test['question_text']]\n\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# more imports for NLP\nfrom nltk.tokenize import word_tokenize\nfrom nltk import pos_tag\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom sklearn.preprocessing import LabelEncoder\nfrom collections import defaultdict\nfrom nltk.corpus import wordnet as wn\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn import model_selection, naive_bayes, svm\nfrom sklearn.metrics import accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# taking backup\ntrainbackup=train\ntestbackup=test\ntrainbackup.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# keeping only 2000 questions for analysis\ntrain= train.head(2000)\ntest= test.head(2000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# step 2 : Tokenization : In this each entry in the corpus will be broken \n#                         into set of words\n\n\ntrain['question_text']= [word_tokenize(entry) for entry in train['question_text']]\n\ntest['question_text']= [word_tokenize(entry) for entry in test['question_text']]\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Set random seed\n# This is used to reproduce the same result every time \n# if the script is kept consistent otherwise each run \n# will produce different results. The seed can be set to any number.\nnp.random.seed(500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## for train data\n\n# step 3, 4 and 5\n# Remove Stop words and Numeric data \n# and perfom Word Stemming/Lemmenting.\n\n# WordNetLemmatizer requires Pos tags to understand if the word is noun or verb\n# or adjective etc. By default it is set to Noun\ntag_map = defaultdict(lambda : wn.NOUN)\ntag_map['J'] = wn.ADJ\ntag_map['V'] = wn.VERB\ntag_map['R'] = wn.ADV\n# the tag_map would map any tag to 'N' (Noun) except\n# Adjective to J, Verb -> v, Adverb -> R\n# that means if you get a Pronoun then it would still be mapped to Noun\n\n\nfor index,entry in enumerate(train['question_text']):\n    # Declaring Empty List to store the words that follow the rules for this step\n    Final_words = []\n    \n    # Initializing WordNetLemmatizer()\n    word_Lemmatized = WordNetLemmatizer()\n    #print(help(pos_tag(entry)))\n    # pos_tag function below will provide the 'tag' \n    # i.e if the word is Noun(N) or Verb(V) or something else.\n    for word, tag in pos_tag(entry):\n    \n        # Below condition is to check for Stop words and consider only \n        # alphabets\n        if word not in stopwords.words('english') and word.isalpha():\n            word_Final = word_Lemmatized.lemmatize(word,tag_map[tag[0]])\n            \n            Final_words.append(word_Final)\n    #print(Final_words)        \n    # The final processed set of words for each iteration will be stored \n    # in 'question_text_final'\n    train.loc[index,'question_text_final'] = str(Final_words)\n    \n   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## for test data\n\n# step 3, 4 and 5\n# Remove Stop words and Numeric data \n# and perfom Word Stemming/Lemmenting.\n\n# WordNetLemmatizer requires Pos tags to understand if the word is noun or verb\n# or adjective etc. By default it is set to Noun\ntag_map = defaultdict(lambda : wn.NOUN)\ntag_map['J'] = wn.ADJ\ntag_map['V'] = wn.VERB\ntag_map['R'] = wn.ADV\n# the tag_map would map any tag to 'N' (Noun) except\n# Adjective to J, Verb -> v, Adverb -> R\n# that means if you get a Pronoun then it would still be mapped to Noun\n\n\nfor index,entry in enumerate(test['question_text']):\n    # Declaring Empty List to store the words that follow the rules for this step\n    Final_words_test = []\n    \n    # Initializing WordNetLemmatizer()\n    word_Lemmatized = WordNetLemmatizer()\n    \n    # pos_tag function below will provide the 'tag' \n    # i.e if the word is Noun(N) or Verb(V) or something else.\n    for word, tag in pos_tag(entry):\n        # Below condition is to check for Stop words and consider only \n        # alphabets\n        if word not in stopwords.words('english') and word.isalpha():\n            word_Final = word_Lemmatized.lemmatize(word,tag_map[tag[0]])\n            Final_words_test.append(word_Final)\n            \n    # The final processed set of words for each iteration will be stored \n    # in 'question_text_final'\n    test.loc[index,'question_text_final'] = str(Final_words_test)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Tfidf_vect = TfidfVectorizer()\nTfidf_vect.fit(train['question_text_final'])\n\nTrain_X_Tfidf = Tfidf_vect.transform(train['question_text_final'])\n\nTest_X_Tfidf = Tfidf_vect.transform(test['question_text_final'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#print(Train_X_Tfidf)\nprint(Test_X_Tfidf[:4])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# You can use the below syntax to see the vocabulary that \n# it has learned from the corpus\nprint(Tfidf_vect.vocabulary_)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Pre-processing is over !!"},{"metadata":{},"cell_type":"markdown","source":"# 5  Use ML Algorithms to Predict the outcome"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\n# fit the training dataset on the NB classifier\nNaive = naive_bayes.MultinomialNB()\n\nNaive.fit(Train_X_Tfidf,train['target'])\n\n# predict the labels on validation dataset\npredictions_NB = Naive.predict(Test_X_Tfidf)\n\n# Use accuracy_score function to get the accuracy\n#print(\"Naive Bayes Accuracy Score -> \",accuracy_score(predictions_NB, train['target'])*100)\nprint(predictions_NB)\n\naccuracy_score(predictions_NB,train.target)*100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Classifier - Algorithm - SVM\n# fit the training dataset on the classifier\nSVM = svm.SVC(C=1.0, kernel='linear', degree=3, gamma='auto')\n\nSVM.fit(Train_X_Tfidf,train['target'])\n\n# predict the labels on validation dataset\npredictions_SVM = SVM.predict(Test_X_Tfidf)\n\n# Use accuracy_score function to get the accuracy\n# print(\"SVM Accuracy Score -> \",accuracy_score(predictions_SVM, train['target'])*100)\nprint(predictions_SVM)\npredictions_SVM[0]\n\n\naccuracy_score(predictions_SVM,train.target)*100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}