{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Import Train and test**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"df_train=pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> **Import libraries**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport re\nimport csv\nimport string\nimport gc\nfrom tqdm import tqdm\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom collections import Counter\nimport matplotlib.pyplot as plt\nfrom wordcloud import WordCloud, STOPWORDS\nfrom scipy.sparse import hstack\nfrom IPython.display import Image\nfrom prettytable import PrettyTable\n\nfrom tqdm import tqdm_notebook\ntqdm_notebook().pandas()\n\nfrom nltk.stem import PorterStemmer, SnowballStemmer, WordNetLemmatizer\nfrom nltk.stem.lancaster import LancasterStemmer\nfrom nltk.util import ngrams","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**EDA of the Quora Questions**"},{"metadata":{},"cell_type":"markdown","source":"**Class Imbalance of toxic and non-toxic**"},{"metadata":{"trusted":true},"cell_type":"code","source":"values = [df_train[df_train['target']==0].shape[0], df_train[df_train['target']==1].shape[0]]\nlabels = ['Non Toxic questions', 'Toxic questions']\n\nplt.pie(values, labels=labels, autopct='%1.1f%%', shadow=True)\nplt.title('Target Distribution')\nplt.tight_layout()\nplt.subplots_adjust(right=1.9)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train.target.value_counts())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Bar graph of class Imbalance**"},{"metadata":{"trusted":true},"cell_type":"code","source":"cnt_srs = df_train['target'].value_counts()\ntrace = go.Bar(\n    x=cnt_srs.index,\n    y=cnt_srs.values,\n    marker=dict(\n        color=cnt_srs.values,\n        colorscale = 'Picnic',\n        reversescale = True\n    ),\n)\n\nlayout = go.Layout(\n    title='Target Count',\n    font=dict(size=18)\n)\n\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig, filename=\"TargetCount\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Feature Extraction of Question from dataset**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Number of words\ndf_train['num_words'] = df_train['question_text'].apply(lambda x: len(str(x).split()))\n#df_test['num_words'] = df_test['question_text'].apply(lambda x: len(str(x).split()))\n\n# Number of capital_letters\ndf_train['num_capital_let'] = df_train['question_text'].apply(lambda x: len([c for c in str(x) if c.isupper()]))\n#df_test['num_capital_let'] = df_test['question_text'].apply(lambda x: len([c for c in str(x) if c.isupper()]))\n\n# Number of special characters\ndf_train['num_special_char'] = df_train['question_text'].str.findall(r'[^a-zA-Z0-9 ]').str.len()\n#df_test['num_special_char'] = df_test['question_text'].str.findall(r'[^a-zA-Z0-9 ]').str.len()\n\n# Number of unique words\ndf_train['num_unique_words'] = df_train['question_text'].apply(lambda x: len(set(str(x).split())))\n#df_test['num_unique_words'] = df_test['question_text'].apply(lambda x: len(set(str(x).split())))\n\n# Number of numerics\ndf_train['num_numerics'] = df_train['question_text'].apply(lambda x: sum(c.isdigit() for c in x))\n#df_test['num_numerics'] = df_test['question_text'].apply(lambda x: sum(c.isdigit() for c in x))\n\n# Number of characters\ndf_train['num_char'] = df_train['question_text'].apply(lambda x: len(str(x)))\n#df_test['num_char'] = df_test['question_text'].apply(lambda x: len(str(x)))\n\n# Number of stopwords\ndf_train['num_stopwords'] = df_train['question_text'].apply(lambda x: len([c for c in str(x).lower().split() if c in STOPWORDS]))\n#df_test['num_stopwords'] = df_test['question_text'].apply(lambda x: len([c for c in str(x).lower().split() if c in STOPWORDS]))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"BoxPlot of the Extracted feature"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_boxplot(_x, _y, _data, _title):\n    sns.boxplot(x=_x, y=_y, data=_data)\n    plt.grid(True)\n    #plt.tick_params(axis='x', which='major', labelsize=15)\n    plt.title(_title,fontsize=17)\n    plt.xlabel(_x, fontsize=10)\n\n# Boxplot: Number of words\nplt.subplot(2, 3, 1)\ndisplay_boxplot('target', 'num_words', df_train, 'No. of words in each class')\n\n# Boxplot: Number of chars\nplt.subplot(2, 3, 2)\ndisplay_boxplot('target', 'num_char', df_train, 'Number of characters in each class')\n\n# Boxplot: Number of unique words\nplt.subplot(2, 3, 3)\ndisplay_boxplot('target', 'num_unique_words', df_train, 'Number of unique words in each class')\n\n# Boxplot: Number of special characters\nplt.subplot(2, 3, 4)\ndisplay_boxplot('target', 'num_special_char', df_train, 'No. of special characters in each class')\n\n# Boxplot: Number of stopwords\nplt.subplot(2, 3, 5)\ndisplay_boxplot('target', 'num_stopwords', df_train, 'Number of stopwords in each class')\n\n# Boxplot: Number of capital letters\nplt.subplot(2, 3, 6)\ndisplay_boxplot('target', 'num_capital_let', df_train, 'No. of capital letters in each class')\n\n\nplt.subplots_adjust(right=3.0)\nplt.subplots_adjust(top=2.0)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Correlation Matrix of Extracted Feature**"},{"metadata":{},"cell_type":"markdown","source":"Many features are extracted and Box plot representation is done to the features, Next Correlation Matrix is plotted to see the correlation between the features."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Correlation matrix\nf, ax = plt.subplots(figsize=(10, 8))\ncorr = df_train.corr()\nsns.heatmap(corr, ax=ax,annot=True)\nplt.title(\"Correlation matrix\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Natural language processing of the question and Visualization of ngrams**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\ntrain1_df = df_train[df_train[\"target\"]==1]\ntrain0_df = df_train[df_train[\"target\"]==0]\n\n## custom function for ngram generation ##\ndef generate_ngrams(text, n_gram=1):\n    token = [token for token in text.lower().split(\" \") if token != \"\" if token not in STOPWORDS]\n    ngrams = zip(*[token[i:] for i in range(n_gram)])\n    return [\" \".join(ngram) for ngram in ngrams]\n\n## custom function for horizontal bar chart ##\ndef horizontal_bar_chart(df, color):\n    trace = go.Bar(\n        y=df[\"word\"].values[::-1],\n        x=df[\"wordcount\"].values[::-1],\n        showlegend=False,\n        orientation = 'h',\n        marker=dict(\n            color=color,\n        ),\n    )\n    return trace\n\n## Get the bar chart from sincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n    for word in generate_ngrams(sent):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(10), 'orange')\n\n## Get the bar chart from insincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n    for word in generate_ngrams(sent):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(10), 'green')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,\n                          subplot_titles=[\"Frequent words of Non-toxic Questions\", \n                                          \"Frequent words of  Toxic questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=500, width=750, paper_bgcolor='rgb(233,233,233)')\npy.iplot(fig, filename='word-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n    for word in generate_ngrams(sent,2):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(10), 'orange')\n\n\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n    for word in generate_ngrams(sent,2):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(10), 'green')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,horizontal_spacing=0.15,\n                          subplot_titles=[\"Frequent bigrams of Non-Toxic Questions\", \n                                          \"Frequent bigrams of Toxic Questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=500, width=750, paper_bgcolor='rgb(233,233,233)')\npy.iplot(fig, filename='word-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"freq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n    for word in generate_ngrams(sent,3):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(10), 'orange')\n\n\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n    for word in generate_ngrams(sent,3):\n        freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(10), 'green')\n\n# Creating two subplots\nfig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04, horizontal_spacing=0.2,\n                          subplot_titles=[\"Frequent trigrams of Non-Toxic \", \n                                          \"Frequent trigrams of Toxic Questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=500, width=800, paper_bgcolor='rgb(233,233,233)')\npy.iplot(fig, filename='word-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\nfrom nltk.corpus import stopwords\nfrom nltk import WordNetLemmatizer\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\nstop_words = set(stopwords.words('english')) \ninsinc_df = df_train[df_train.target==1]\nsinc_df = df_train[df_train.target==0]\ndef plot_ngrams(n_grams):\n    ## custom function for ngram generation ##\n    def generate_ngrams(text, n_gram=1):\n        token = [token for token in text.lower().split(\" \") if token != \"\" if token not in stop_words]\n        ngrams = zip(*[token[i:] for i in range(n_gram)])\n        return [\" \".join(ngram) for ngram in ngrams]\n    ## custom function for horizontal bar chart ##\n    def horizontal_bar_chart(df, color):\n        trace = go.Bar(\n            y=df[\"word\"].values[::-1],\n            x=df[\"wordcount\"].values[::-1],\n            showlegend=False,\n            orientation = 'h',\n            marker=dict(\n                color=color,\n            ),\n        )\n        return trace\n    def get_bar(df, bar_color):\n        freq_dict = defaultdict(int)\n        for sent in df[\"question_text\"]:\n            for word in generate_ngrams(sent, n_grams):\n                freq_dict[word] += 1\n        fd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\n        fd_sorted.columns = [\"word\", \"wordcount\"]\n        trace = horizontal_bar_chart(fd_sorted.head(13), bar_color)\n        return trace    \n    trace0 = get_bar(sinc_df, 'green')\n    trace1 = get_bar(insinc_df, 'red')\n    # Creating two subplots\n    if n_grams == 1:\n        wrd = \"words\"\n    elif n_grams == 2:\n        wrd = \"Bigrams\"\n    elif n_grams == 3:\n        wrd = \"Trigrams\"    \n    fig = toolsmake_subplots(rows=1, cols=2, vertical_spacing=0.03,subplot_titles=[\"Frequent \" + wrd + \" of Toxic\", \n            \"Frequent \" + wrd + \" of Non-toxic \"])\n    fig.append_trace(trace0, 1, 1)\n    fig.append_trace(trace1, 1, 2)\n    fig['layout'].update(height=500, width=750, paper_bgcolor='rgb(233,233,233)', title=wrd + \" Count Plots\")\n    py.iplot(fig, filename='word-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Unigram\nplot_ngrams(1)\n#Bigram\nplot_ngrams(2)\n#Trigram\nplot_ngrams(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}