{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51447dabbf4197832a5f36709c6fce0565b18d83"},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport nltk\nfrom nltk.tokenize import word_tokenize\nfrom nltk import FreqDist\nfrom nltk.corpus import stopwords\nimport seaborn as sns\nimport re\nfrom nltk.stem import SnowballStemmer,WordNetLemmatizer\nstemmer=SnowballStemmer('english')\nlemma=WordNetLemmatizer()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11d64eaa187e44f25d6928bde3c60e157c993a0e"},"cell_type":"code","source":"print(\"Size of training data:\",train_data.size)\nprint(\"Columns in the traing data\",train_data.columns)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"edb5cfc6cdf30cbcc4ae6c9b7b0018637072e5ab"},"cell_type":"markdown","source":"## sample of the question"},{"metadata":{"trusted":true,"_uuid":"ba7033b2fb647b8472edf87d5ca9ea745f184cbe"},"cell_type":"code","source":"print(\"question_text ->\",train_data.iloc[1]['question_text'],\"Target:->\",train_data.iloc[1]['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c663f4c9343e8bf939f13046554cdbde156a5e19"},"cell_type":"code","source":"target_count = train_data[['qid','target']].groupby(['target']).agg('count').reset_index()\ntarget_count.columns = [\"target\",\"Count\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"81a575a684251d0e62f5f8d49479a094b391ac3d"},"cell_type":"code","source":"labels = ['Sincere Questions','Insincere Questions']  \nfig1, ax1 = plt.subplots()\ncolors = ['#66b3ff','#ff9999']\nax1.pie(target_count[\"Count\"],labels=labels,colors=colors, autopct='%1.1f%%',shadow=True, startangle=90)\n# Equal aspect ratio ensures that pie is drawn as a circle\nax1.axis('equal')  \nplt.tight_layout()\nplt.show()\nprint(\"total sincere questions\",target_count.iloc[0][\"Count\"])\nprint(\"total insincere questions\",target_count.iloc[1][\"Count\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb58a2e9f872ef258c28475c3a9b28a7b4ceaf0e"},"cell_type":"markdown","source":"## analazing insincere questions"},{"metadata":{"trusted":true,"_uuid":"debb93f894b97351ed40819cbb7220eb529ec995"},"cell_type":"code","source":"insincere_question_df = train_data[['question_text']][train_data['target'] == 1]\nsincere_question_df = train_data[['question_text']][train_data['target'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b59963a0899564ae1b39c66daa4b57930a986c75","scrolled":true},"cell_type":"code","source":"insincere_question_df.size","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b449beb463dc601072061f854f6a989aa1838078"},"cell_type":"markdown","source":"## most frequent occured word in insincere question"},{"metadata":{"trusted":true,"_uuid":"c083a3d8742f43d64b191fe7bf376a86f3e9a220"},"cell_type":"code","source":"def text_clean(review_col):\n    stops = set(stopwords.words(\"english\"))\n    stops.update(['would','many','u','much','more'])\n    text_corpus=[]\n    for i in range(0,len(review_col)):\n        review=str(review_col[i])\n        review=re.sub('[^a-zA-Z]',' ',review)\n        word_token = word_tokenize(str(review).lower())\n        #review = [word for word in word_token if word not in stops]\n        #review=' '.join(review)\n        review=[lemma.lemmatize(w) for w in word_token if w not in stops]\n        review=' '.join(review)\n        text_corpus.append(review) \n    return text_corpus","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb87ffb2897f2a0ac9c1095223d84ce68cfc2d25"},"cell_type":"code","source":"insincere_question_df['question_word']=text_clean(insincere_question_df['question_text'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06bcf3a60aeee9afe7266cf7a35beb5e4b24f5f1"},"cell_type":"code","source":"insincere_question_clean_word = insincere_question_df.question_word.str.split(expand=True).stack().value_counts().to_frame()\ninsincere_question_clean_word.reset_index(inplace=True)\ninsincere_question_clean_word.columns = ['word','count']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9da51109e5f39784096ca323350df3ab56c5556"},"cell_type":"code","source":"sns.set(style=\"white\")\n# Plot word and it's count for top 50 words\nsns.relplot(x=\"count\", y=\"word\", size=\"count\",\n            sizes=(40, 400), alpha=.5, palette=\"muted\",\n            height=8, data=insincere_question_clean_word[0:20])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6798b411d27e87e3436397d372a09a2f4abef1f"},"cell_type":"markdown","source":"## most frequent occured word in Sincere question"},{"metadata":{"trusted":true,"_uuid":"4349c52450e25e595c845c98b11217e36c746c6d"},"cell_type":"code","source":"sincere_question_df['question_word']=text_clean(sincere_question_df['question_text'].values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"180b9db61ad02b16f84031673639c590ca61b3e3"},"cell_type":"code","source":"sincere_question_clean_word = sincere_question_df.question_word.str.split(expand=True).stack().value_counts().to_frame()\nsincere_question_clean_word.reset_index(inplace=True)\nsincere_question_clean_word.columns = ['word','count']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f1e89b2e9ca97618847a21e1dbd46d8174367a0"},"cell_type":"code","source":"sns.set(style=\"white\")\n# Plot word and it's count for top 50 words\nsns.relplot(x=\"count\", y=\"word\", size=\"count\",\n            sizes=(40, 400), alpha=.5, palette=\"muted\",\n            height=8, data=sincere_question_clean_word[0:20])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5c71b0591d8f896d2001bcb85f22edb9ac84e220"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}