{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\n\n\n#train_char = pd.read_csv('../input/train.csv')\n#test_char = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\nfrom nltk.corpus import stopwords\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\nstop_words = set(stopwords.words('english')) \ninsinc_df = train[train.target==1]\nsinc_df = train[train.target==0]\n\ndef plot_ngrams(n_grams):\n\n    ## custom function for ngram generation ##\n    def generate_ngrams(text, n_gram=1):\n        token = [token for token in text.lower().split(\" \") if token != \"\" if token not in stop_words]\n        ngrams = zip(*[token[i:] for i in range(n_gram)])\n        return [\" \".join(ngram) for ngram in ngrams]\n\n    ## custom function for horizontal bar chart ##\n    def horizontal_bar_chart(df, color):\n        trace = go.Bar(\n            y=df[\"word\"].values[::-1],\n            x=df[\"wordcount\"].values[::-1],\n            showlegend=False,\n            orientation = 'h',\n            marker=dict(\n                color=color,\n            ),\n        )\n        return trace\n\n    def get_bar(df, bar_color):\n        freq_dict = defaultdict(int)\n        for sent in df[\"question_text\"]:\n            for word in generate_ngrams(sent, n_grams):\n                freq_dict[word] += 1\n        fd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\n        fd_sorted.columns = [\"word\", \"wordcount\"]\n        trace = horizontal_bar_chart(fd_sorted.head(10), bar_color)\n        return trace    \n\n    trace0 = get_bar(sinc_df, 'blue')\n    trace1 = get_bar(insinc_df, 'blue')\n\n    # Creating two subplots\n    if n_grams == 1:\n        wrd = \"words\"\n    elif n_grams == 2:\n        wrd = \"bigrams\"\n    elif n_grams == 3:\n        wrd = \"trigrams\"\n    \n    fig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,\n                              subplot_titles=[\"Frequent \" + wrd + \" of sincere questions\", \n                                              \"Frequent \" + wrd + \" of insincere questions\"])\n    fig.append_trace(trace0, 1, 1)\n    fig.append_trace(trace1, 1, 2)\n    fig['layout'].update(height=500, width=1150, paper_bgcolor='rgb(233,233,233)', title=wrd + \" Count Plots\")\n    py.iplot(fig, filename='word-plots')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_ngrams(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_ngrams(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plot_ngrams(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#######################\n#EDA for training Data#\n#######################\n\nt1 = train[[\"target\"]]\ndef score_to_numeric(x):\n    if x==0:\n        return \"Sincere\"\n    if x==1:\n        return \"Insincere\"\nt1['target_class'] = t1['target'].apply(score_to_numeric)\nt1_1 = t1.groupby(['target_class']).count()\nt1_1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"########################\n#number of group target#\n########################\n\nimport plotly.graph_objects as go\nfig = go.Figure(go.Bar(\n            x=[80810, 1225312],\n            y=['Insincere', 'Sincere'],\n            orientation='h',\n    marker=dict(\n        color='rgba(51, 204, 204, 0.6)',\n        line=dict(color='rgba(58, 71, 80, 1.0)', width=3))))\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer\n####################################\n#number of character in each target#\n####################################\nt2 = train[[\"question_text\", \"target\"]]\ndef score_to_numeric(x):\n    if x==0:\n        return \"Sincere\"\n    if x==1:\n        return \"Insincere\"\nt2['target_class'] = t2['target'].apply(score_to_numeric)\nt2_1 = t2[['question_text', 'target_class']]\nt2_1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tokenize_character(text):\n    text = text.encode('ascii', 'ignore').decode('ascii')\n    text = text.lower()   \n    text = re.sub(r'\\b(?:(?:https?|ftp)://)?\\w[\\w-]*(?:\\.[\\w-]+)+\\S*', ' ', text) # remove hyperlink,subs charact in the brackets\n    text = re.sub(\"[\\r\\n]\", ' ', text) # remove new line characters\n    tokens = word_tokenize(text)\n    tokens = [token for token in tokens if re.match(r'.*[a-z]{2,}.*', token)]\n    return tokens\n\n#t2_1[\"tokens\"] = t2_1['question_text'].map(lambda x: tokenize_character(x))\nt2_1[\"tokens\"] = t2_1['question_text'].apply(tokenize_character) \nt2_1[\"word_count\"] = t2_1.tokens.apply(lambda x: len(x))\nt2_1.tail()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t2_2 = t2_1[[\"target_class\", \"word_count\"]]\nt2_2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# library & dataset\nimport seaborn as sns\n#df = sns.load_dataset('iris')\nsns.boxplot(x=t2_2[\"target_class\"], y=t2_2[\"word_count\"])\n#sns.plt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}