{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading Data set"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train=pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test=pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\")\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.count()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Word Cloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS \nimport matplotlib.pyplot as plt ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#            Word Cloud for whole training data set."},{"metadata":{"trusted":true},"cell_type":"code","source":"text1 = \" \".join(review for review in train.question_text)\nwordcloud = WordCloud(background_color=\"white\").generate(text1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here,\n   Will,best,India,make,use,one,think,good are repeated most of the time."},{"metadata":{},"cell_type":"markdown","source":"# Word cloud for Negative Response test data set"},{"metadata":{"trusted":true},"cell_type":"code","source":"Negative_response = \" \".join(review for review in train[train[\"target\"]==0].question_text)\nwordcloud0 = WordCloud(background_color=\"white\", \n                          mode=\"RGBA\", max_words=1000,).generate(Negative_response)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(wordcloud0, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Here is surprise for me....\n   For Negative response some word like best,good,use are repeated most of the time,which prety good."},{"metadata":{},"cell_type":"markdown","source":"# Word cloud for Positive Response of traing data set"},{"metadata":{"trusted":true},"cell_type":"code","source":"Positive_response = \" \".join(review for review in train[train[\"target\"]==1].question_text)\nwordcloud1 = WordCloud(background_color=\"red\", \n                          mode=\"RGBA\", max_words=1000,).generate(Positive_response)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.imshow(wordcloud1, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Word Frequency plot of sincere**"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom yellowbrick.text import FreqDistVisualizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer = CountVectorizer(stop_words='english')\ndocs       = vectorizer.fit_transform(text for text in train['question_text'])\nfeatures   = vectorizer.get_feature_names()\n\nvisualizer = FreqDistVisualizer(\n    features=features#, size=(1080, 720)\n)\nvisualizer.fit(docs)\nvisualizer.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# It's very big data and I can't run any algorithm due to memory size but simple coding available for beginner with statistical background."},{"metadata":{"trusted":true},"cell_type":"markdown","source":"import re\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer \n\nstops = stopwords.words('english')\n\ncorpus = []\nfor i in range(0, 1306122):\n    question_text = re.sub('[^a-zA-Z]', ' ', train['question_text'][i])\n    question_text = question_text.lower()\n    question_text = question_text.split()\n    ps = PorterStemmer()\n    question_text = [ps.stem(word) for word in question_text if not word in set(stops)]\n    question_text = ' '.join(question_text)\n    corpus.append(question_text)\n\nfrom sklearn.feature_extraction.text import CountVectorizer\ncv = CountVectorizer(max_features = 1500)\nX = cv.fit_transform(corpus).toarray()\ny = train.iloc[:,train.columns==\"target\"]"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"from sklearn.model_selection import train_test_split \nX_train, X_test, y_train, y_test = train_test_split(X, y,test_size = 0.3, \n                                                    random_state=1,\n                                                    stratify=y)\n\nfrom xgboost import XGBClassifier\nclf=XGBClassifier(random_state=2020,n_jobs=-1)\nclf.fit(X_train,y_train)\ny_pred=clf.predict(X_test)\nfrom sklearn import metrics\nprint(\"Accuracy of XGboost Algorithm model is:\",\nmetrics.accuracy_score(y_test, y_pred)*100)\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}