{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # data visiualization\nimport seaborn as sns # data visisualization like distribytion chart, matrix plot, heat maps\nimport sklearn # scikit library for machine learning\n\n!pip install altair\n!pip install datapane\nimport altair as alt # declarative statistical visualization library for Python, based on Vega and Vega-Lite.\nimport datapane as dp # open source framework which makes it easy to build and share reports","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":false},"cell_type":"markdown","source":"# 1. Data set import****"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 2. Data Preprocessing"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"training_data = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest_data = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\nsubmission_data=pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"print('Training Data',training_data.info()) # find records and attributes\n'\\n'\nprint('Training data column headings:',training_data.columns) # column headings\n'\\n'\nprint('Training data shape:',training_data.shape) # Rows & columns\n'\\n'\nprint('Training Data',training_data.head()) #top 5 records\n'\\n'\nprint('Sample Training data',training_data.sample(10)) # sample 10 records","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"print('Test Data information',test_data.info()) # find records and attributes\n'\\n'\nprint('Test data column headings:',test_data.columns) # column headings\n'\\n'\nprint('Test data shape:',test_data.shape) # Rows & columns\n'\\n'\nprint('Test Data',test_data.head()) #top 5 records\n'\\n'\nprint('Sample Test data',test_data.sample(10)) # sample 10 records","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Checking insincere and sincere message content"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"training_data[training_data.target==1][:5]  # top 5 labeled insincere questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"training_data[training_data.target==0][:5] # top 5 labeled sincere questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#Frequency count of sincere and insincere question texts:\ncount=training_data['target'].value_counts()\nprint('Total Counts of both sets'.format(),count)\n\nplt.figure(figsize=(15,5))\nsns.countplot(y=\"target\",\n              palette =['green','red'],\n              data=training_data)\nplt.suptitle(\"Frequency of Sincere Questions (0) & Insincere Questions (1)\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## The Training dataset is unbalanced more towards sincere questions"},{"metadata":{},"cell_type":"markdown","source":"# 3. Exploratory data analysis"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"sns.distplot(training_data['target'].value_counts(),\n            kde=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"insincere_label=training_data[training_data['target']== 1]['question_text']\nprint(insincere_label.head())  #top insincere questions\n\n\nsincere_label=training_data[training_data['target']== 0]['question_text']\nprint(sincere_label.head())  #top sincere questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#Frequency of insincere and sincere questions\n#Function for checking word length\ndef freq_len(data):\n    return len(data)\n\n\nfreq_sincere = sincere_label.str.split().apply(lambda z:freq_len(z))\nprint(\"Sincere Questions Length:\" + str(freq_sincere))\n\nfreq_insincere = insincere_label.str.split().apply(lambda z:freq_len(z))\nprint(\"Insincere Questions Length:\" + str(freq_insincere))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#Visualizing distributions of length of insincere and sincere questions in the entire training data\n\nfig,axes=plt.subplots(1,2)\n\nsns.distplot(freq_insincere,ax=axes[0],color='red')\n\nsns.distplot(freq_sincere,ax=axes[1],color='green')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Clearly we see people using more words in insincere questions than in sincere questions"},{"metadata":{},"cell_type":"markdown","source":"# Frequency count of Punctuations"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"import string","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"freq_sincere_punctuations= sincere_label.apply(lambda z: len([c for c in str(z) if c in string.punctuation])) #punctuation in insincere questions\n\nfreq_insincere_punctuations= insincere_label.apply(lambda z:len([c for c in str(z) if c in string.punctuation])) #punctuations in sincere questions\n\n#Distribution plot for length of punctuations in insincere and sincere questions\n\nfig,axes=plt.subplots(1,2)\n\nsns.distplot(freq_insincere_punctuations,ax=axes[0],color='red')\n\nsns.distplot(freq_sincere_punctuations,ax=axes[0],color='green')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Frequency Count of Stopwords"},{"metadata":{},"cell_type":"markdown","source":"### Importing nltk library and stopwords"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"import re\nimport nltk\nnltk.download('stopwords')\nfrom nltk.corpus import stopwords","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"stop_words=set(stopwords.words('english')) #designating stopwords\n\nfreq_insincere_stops= insincere_label.apply(lambda z : np.mean([len(z) for w in str(z).split()])) #stopwords in insincere questions\n\nfreq_sincere_stops= sincere_label.apply(lambda z : np.mean([len(z) for w in str(z).split()])) #stopwords in sincere questions\n\n#Distribution plot for stopwords in insincere and sincere questions\n\nfig,axes=plt.subplots(1,2)\n\nsns.distplot(freq_insincere_stops,ax=axes[0],color='red')\n\nsns.distplot(freq_sincere_stops,ax=axes[1],color='green')\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualizing occurence of words though word clouds"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"!pip install wordcloud\nimport wordcloud\nfrom wordcloud import WordCloud, STOPWORDS ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"def display_cloud(data,color,figsize):\n    plt.subplots(figsize=figsize)\n    wc = WordCloud(stopwords=STOPWORDS,\n                   background_color=\"white\", \n                   contour_width=2, \n                   contour_color=color,\n                   max_words=2000, \n                   max_font_size=256,\n                   random_state=42)\n    wc.generate(' '.join(data))\n    plt.imshow(wc, interpolation=\"bilinear\")\n    plt.axis('off')\n    plt.show()\n    \ndisplay_cloud(training_data['question_text'],color='red',figsize=(15,15)) #WordCloud for the training daata","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"display_cloud(insincere_label,'red',figsize=(15,15))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"display_cloud(sincere_label,'green',figsize=(15,15)) #Word cloud for sincere questions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Most frequently occuring words in descending order"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from collections import Counter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#Simplified counter function\ndef create_corpus(x=0):\n    corpus=[]\n    for x in training_data[training_data['target']==x]['question_text'].str.split():\n        for i in x:\n            corpus.append(i)\n    return corpus\n\ncorpus=create_corpus(x=0)\ncounter=Counter(corpus)\nmost=counter.most_common()\nx=[]\ny=[]\nfor word,count in most[:100]:\n    if (word not in stop_words) :\n        x.append(word)\n        y.append(count)\n        \nplt.figure(figsize=(15,10))\nsns.barplot(x=y,y=x)\nplt.title(\"Most frequent words in descending order\")\nplt.xlabel(\"frequency\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#sincere_wordcloud = display_cloud(sincere_label,'green',figsize=(15,15))\n#insincere_wordcloud = display_cloud(insincere_label,'green',figsize=(15,15))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#Publishing descriptive analysis results\n\n#dp.Report(\n#    dp.Plot(plot), \n#    dp.Table(df)\n#).publish(name='Covid Report', open=True) \n\n#dp.Report(\n#    dp.Plot(sincere_wordcloud),\n#    dp.Plot(insincere_wordcloud)\n#).publish(name='Quora Report', open=True) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predictive analysis"},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"from nltk.stem.porter import PorterStemmer\ncorpus = []\nfor i in range(0, 1306122):\n    review = re.sub('[^a-zA-Z]', ' ', training_data['question_text'][i])\n    review = review.lower()\n    review = review.split()\n    ps = PorterStemmer()\n    review = [ps.stem(word) for word in review if not word in set(stopwords.words('english'))]\n    review = ' '.join(review)\n    corpus.append(review)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Creating the Bag of Words model\nfrom sklearn.feature_extraction.text import CountVectorizer\ncv = CountVectorizer(max_features = 1500)\nX = cv.fit_transform(corpus).toarray()\ny = training_data.iloc[:, -1].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Splitting the dataset into the Training set and Test set\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.20, random_state = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Fitting Naive Bayes to the Training set\nfrom sklearn.naive_bayes import GaussianNB\nclassifier = GaussianNB()\nclassifier.fit(X_train, y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Predicting the Test set results\ny_pred = classifier.predict(X_test)\ny_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"# Making the Confusion Matrix\nfrom sklearn.metrics import confusion_matrix\ncm = confusion_matrix(y_test, y_pred)\ncm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"#Publishing output of analysis\n#dp.Report(\n#    dp.Plot(plot), \n#    dp.Table(df)\n#).publish(name='Covid Report', open=True) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}