{"cells":[{"metadata":{"_uuid":"62cddbb3de5d738203f8dc9787eb95239b5fd1f5"},"cell_type":"markdown","source":"# Intro\n\nThere are so many insincere and inaccurate questions I saw on Quora. Even the questioners themselves not sure about what they are asking. I'm here to explore the questions text and implement text mining to find a way to build NLP Model classify the insincere question from sincere one."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nplt.style.use('ggplot')\n\nfrom wordcloud import WordCloud, ImageColorGenerator\nfrom PIL import Image\n\nimport datetime\nfrom string import punctuation","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"939440229757322f5ba411c75f546244bec8d34f"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e656b878930067e291030064479a0be9358168b5"},"cell_type":"code","source":"sum(train.target)/len(train)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"826ab06ec1ab56e74f67c477b8bede55fea74131"},"cell_type":"markdown","source":"### Only 6.18% of the data are target 1"},{"metadata":{"trusted":true,"_uuid":"a7cf838e28d88bd243feabb5395c23d0a335d2b1"},"cell_type":"code","source":"train = train.drop('qid',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b67e9c8f5fbb336df1b9dff6e43ec471910ac676"},"cell_type":"code","source":"# preprocess text for further use\ntrain['question_text'] = train.question_text.apply(lambda x: x.lower())\ntrain['question_text'] = train.question_text.apply(lambda x: ''.join([c for c in x if c not in punctuation]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c5ba3258deb2ac0e2f5fd302fedf9f2e331caec"},"cell_type":"code","source":"train['question_length'] = train.question_text.apply(lambda x: len(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2249a2690566764bb34e70c06b3717180af4fa35"},"cell_type":"code","source":"print('The max length of question is:',train.question_length.max())\nprint('The minimum length of question is:',train.question_length.min())\nprint('The mean length of question is:',train.question_length.mean())\nprint('The max standard deviation of question is:',train.question_length.std())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9497f8f06a4bb050be5322b247ae255756b8183"},"cell_type":"code","source":"train.question_length.hist(bins=50)\nplt.title('The distribution of length of questions')\nplt.axvline(np.mean(train.question_length),color='y')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"542892f8eb3aceb252b88e456b958d967c6db151"},"cell_type":"code","source":"train['question_length_scaled'] = train.question_length.apply(lambda x: np.log(x+1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c22b374338e0a85343e4998a08139f61b596283"},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"5ad8b14e58ceefff912158351678f8a9fd8ffdcb"},"cell_type":"code","source":"train.question_length_scaled.hist(bins=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbe4abdeb71a7c832e1306550995ec18103157ae"},"cell_type":"code","source":"sns.boxplot(train.target,train.question_length_scaled)\nplt.title('The distribution of length of target or not')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8a258375ed2907037101563d6963cbf21851292c"},"cell_type":"code","source":"Q = np.array(Image.open('../input/quora-logo1/quora-logo-rubber-stamp.png'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b478f296f95a7dc8cbfb73a9bf5ee76f6e6fa8c2"},"cell_type":"code","source":"np.random.seed(321)\nsns.set(rc={'figure.figsize':(14,8)})\nreviews = ' '.join(train['question_text'].tolist())\n\nwordcloud = WordCloud(mask=Q,background_color=\"white\").generate(reviews)\nplt.figure()\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis(\"off\")\nplt.margins(x=0, y=0)\nplt.title('Questions',size=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9351fab55d5134bd341db4290ce0524d2ea570ca"},"cell_type":"markdown","source":"I'll split the data into target 1 and target 0 to see what's different in their word clouds."},{"metadata":{"trusted":true,"_uuid":"a588c93805cda431b56319e8d28aabf582170c70"},"cell_type":"code","source":"train_1 = train[train.target == 1]\ntrain_0 = train[train.target == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1022eea435cc161d46222d5db439bee27edb57e"},"cell_type":"code","source":"reviews = ' '.join(train_1['question_text'].tolist())\n\nwordcloud = WordCloud(mask=Q,background_color=\"white\").generate(reviews)\nplt.figure()\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis(\"off\")\nplt.margins(x=0, y=0)\nplt.title('Target 1',size=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8eae2e3e682b857f7bf270f9070fd04245f9081a"},"cell_type":"markdown","source":"What I saw is:\n\n* Donal Trump\n* liberal\n* india\n* muslim\n* american\n* christian\n* conservative"},{"metadata":{"trusted":true,"_uuid":"b1f174d55fb2c16a0c3d3e96d5b9798903bb1e3a"},"cell_type":"code","source":"reviews = ' '.join(train_0['question_text'].tolist())\n\nwordcloud = WordCloud(mask=Q,background_color=\"white\").generate(reviews)\nplt.figure()\nplt.imshow(wordcloud, interpolation=\"bilinear\")\nplt.axis(\"off\")\nplt.margins(x=0, y=0)\nplt.title('Target 0',size=20)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95992e7816f23a98538cae739c858c7cd4272dc3"},"cell_type":"markdown","source":"One thing that catch my eyes is that both of the word clouds have india. But the ratios are different. While in target 1, there are trump, liberal, muslim, and american in there. I'll use counter to find more insights."},{"metadata":{"trusted":true,"_uuid":"5241d2a6bf378a09b33458b4f4ba1d84efc756d8"},"cell_type":"code","source":"from collections import Counter\n\ntext = ' '.join(train['question_text'].tolist())\nquestion_word = text.split(' ')\nall_question = ' '.join(question_word)\nwords = all_question.split()\n\n# words wrong datatype\ncounts = Counter(words)\nvocab = sorted(counts, key=counts.get, reverse=True)\nvocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n\nquestions_ints = []\nfor questions in question_word:\n    questions_ints.append([vocab_to_int[word] for word in questions.split()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bf24af3e2589bfa021f2e3d65a372b77c564e98"},"cell_type":"code","source":"print('Unique words: ', len((vocab_to_int)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73f74232c56c0488d59f36b6b98d2607b34bbda0"},"cell_type":"code","source":"counts.most_common(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b0af4dfa8351edc3378107c96e2b9399c47b351"},"cell_type":"code","source":"text = ' '.join(train_0['question_text'].tolist())\nquestion_word = text.split(' ')\nall_question = ' '.join(question_word)\nwords = all_question.split()\n\n# words wrong datatype\ncounts = Counter(words)\nvocab = sorted(counts, key=counts.get, reverse=True)\nvocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n\nquestions_ints = []\nfor questions in question_word:\n    questions_ints.append([vocab_to_int[word] for word in questions.split()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb3ad84a1c58877634c4aa05ac8c47c2d4d172a1"},"cell_type":"code","source":"counts.most_common(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10488d02f2cb7871eae8c4fad503ac01a8601114"},"cell_type":"code","source":"text = ' '.join(train_1['question_text'].tolist())\nquestion_word = text.split(' ')\nall_question = ' '.join(question_word)\nwords = all_question.split()\n\n# words wrong datatype\ncounts = Counter(words)\nvocab = sorted(counts, key=counts.get, reverse=True)\nvocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n\nquestions_ints = []\nfor questions in question_word:\n    questions_ints.append([vocab_to_int[word] for word in questions.split()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a5633ab2147deb65bc4da77cf7e3a410183a7d1"},"cell_type":"code","source":"counts.most_common(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2501d54057624be686953dbec30e08c049439b37"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fccadf31a83a615fdbfb7c3398008ef21f6f6293"},"cell_type":"markdown","source":"As you see, the most of the words are not really meaningful to us."},{"metadata":{"trusted":true,"_uuid":"8f09b958c1e2ebe2888fc78aa60c85478a37424d"},"cell_type":"code","source":"from nltk.corpus import stopwords \nfrom nltk.tokenize import word_tokenize \n\nstop_words = set(stopwords.words('english')) \n\ntrain['question_text'] = train.question_text.apply(lambda x: word_tokenize(x))\n\ntrain['question_text'] = train.question_text.apply(lambda x: [w for w in x if w not in stop_words])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c77ee9f46f9d1e110be0cf90aae04ea0bfbb22d0"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cf97fe0806ec9a2b2e6720929c9b53c93fb2e3a"},"cell_type":"code","source":"train['question_text'] = train.question_text.apply(lambda x: ' '.join(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3a4b4f9b426dcd2a0031023915d683a81e1d9f9"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80effe6efb8adef31c4e3aedfad1b9785a66411b"},"cell_type":"code","source":"text = ' '.join(train['question_text'].tolist())\nquestion_word = text.split(' ')\nall_question = ' '.join(question_word)\nwords = all_question.split()\n\n# words wrong datatype\ncounts = Counter(words)\nvocab = sorted(counts, key=counts.get, reverse=True)\nvocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n\nquestions_ints = []\nfor questions in question_word:\n    questions_ints.append([vocab_to_int[word] for word in questions.split()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa99c093db4e4d9bc48b1d67828032ab1e5af316"},"cell_type":"code","source":"print('Unique words: ', len((vocab_to_int)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1ba876b4e1c35c16e5c769e6de8c55aab808aa52"},"cell_type":"code","source":"counts.most_common(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d18443babceb5c7d9f39ff47cafb6c75d45a2fb"},"cell_type":"code","source":"train['question_text'] = train['question_text'].apply(lambda x: x.replace('’', \"\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd24cdcf708e7c9c010682c7510c050d469aa34c"},"cell_type":"code","source":"train_1 = train[train.target == 1]\ntrain_0 = train[train.target == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdc78fce26971b9933406292a95b1483600f22e3"},"cell_type":"code","source":"text = ' '.join(train_0['question_text'].tolist())\nquestion_word = text.split(' ')\nall_question = ' '.join(question_word)\nwords = all_question.split()\n\n# words wrong datatype\ncounts = Counter(words)\nvocab = sorted(counts, key=counts.get, reverse=True)\nvocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n\nquestions_ints = []\nfor questions in question_word:\n    questions_ints.append([vocab_to_int[word] for word in questions.split()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b09ab8bfe9eae7becf8fa9e44d2e3f79e3eba7c6"},"cell_type":"code","source":"counts.most_common(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54df67cfab2bfdd12c1ce2395fca59f4be7abd4c"},"cell_type":"code","source":"text = ' '.join(train_1['question_text'].tolist())\nquestion_word = text.split(' ')\nall_question = ' '.join(question_word)\nwords = all_question.split()\n\n# words wrong datatype\ncounts = Counter(words)\nvocab = sorted(counts, key=counts.get, reverse=True)\nvocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n\nquestions_ints = []\nfor questions in question_word:\n    questions_ints.append([vocab_to_int[word] for word in questions.split()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4e6f6bf897cdf57e6da5326d28418b3217a4b6b6","scrolled":true},"cell_type":"code","source":"counts.most_common(20)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7c0e10afb175fde8926b1f698312648edecd047a"},"cell_type":"markdown","source":"Well, this is a good result. I can tell the difference of target 1 from target 0.\n\nBut I'll remove the duplicated words in both part"},{"metadata":{"trusted":true,"_uuid":"24c47998367eaee269b3a9b677ea90285cd27aa2"},"cell_type":"code","source":"dup_words = ['people','would','get','like','india','think', 'many']\n\ntrain['question_text'] = train.question_text.apply(lambda x: word_tokenize(x))\n\ntrain['question_text'] = train.question_text.apply(lambda x: [w for w in x if w not in dup_words])\n\ntrain['question_text'] = train.question_text.apply(lambda x: ' '.join(x))\n\ntrain_1 = train[train.target == 1]\ntrain_0 = train[train.target == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a76abec1dd76d30a62f09c0e1d3df2054ab04236"},"cell_type":"code","source":"text = ' '.join(train_0['question_text'].tolist())\nquestion_word = text.split(' ')\nall_question = ' '.join(question_word)\nwords = all_question.split()\n\n# words wrong datatype\ncounts = Counter(words)\nvocab = sorted(counts, key=counts.get, reverse=True)\nvocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n\nquestions_ints = []\nfor questions in question_word:\n    questions_ints.append([vocab_to_int[word] for word in questions.split()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"784e16e050799d7fd7bb901bc0943be399dc5bd3"},"cell_type":"code","source":"counts.most_common(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f52ad4e655a4b4d566bf90cb962948c0171fdb61"},"cell_type":"code","source":"text = ' '.join(train_1['question_text'].tolist())\nquestion_word = text.split(' ')\nall_question = ' '.join(question_word)\nwords = all_question.split()\n\n# words wrong datatype\ncounts = Counter(words)\nvocab = sorted(counts, key=counts.get, reverse=True)\nvocab_to_int = {word: ii for ii, word in enumerate(vocab, 1)}\n\nquestions_ints = []\nfor questions in question_word:\n    questions_ints.append([vocab_to_int[word] for word in questions.split()])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f530730d5861cc7a2fffaad597e1a83c1e44b43"},"cell_type":"code","source":"counts.most_common(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"416b5ba50a31122d0539221cea9e1ba79a0fd5ab"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import train_test_split\n\nfrom sklearn.feature_selection import SelectKBest\nfrom sklearn.feature_selection import chi2\n\nX_train, X_test, y_train, y_test = train_test_split(train[\"question_text\"], train['target'], test_size=0.33\n                                    ,random_state=53)\n\n# Initialize a CountVectorizer object: count_vectorizer\ncount_vectorizer = CountVectorizer(stop_words=\"english\")\n\n# Transform the training data using only the 'text' column values: count_train \ncount_train = count_vectorizer.fit_transform(X_train)\n\ny_train = np.asarray(y_train.values)\n\nch2 = SelectKBest(chi2, k = 300)\n\nX_new = ch2.fit_transform(count_train, y_train)\n\n# Transform the test data using only the 'text' column values: count_test \ncount_test = count_vectorizer.transform(X_test)\n\nX_test_new = ch2.transform(X=count_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad2599fb26c1bcd0ba9099c03066202283eb6762"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\n\n# Initialize a TfidfVectorizer object: tfidf_vectorizer\ntfidf_vectorizer = TfidfVectorizer(stop_words=\"english\", max_df=0.7)\n\n# Transform the training data: tfidf_train \ntfidf_train = tfidf_vectorizer.fit_transform(X_train)\n\n# Transform the test data: tfidf_test \ntfidf_test = tfidf_vectorizer.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18dcc894486ba3fbcfe6e95606fbd6b32f6dbcfb"},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn import metrics\n\nclf = RandomForestClassifier()\n# Fit the classifier to the training data\nclf.fit(X_new, y_train)\n\n# Create the predicted tags: pred\npred = clf.predict(X_test_new)\n\n# Calculate the accuracy score: score\nscore = metrics.accuracy_score(y_test, pred)\nprint('Accuracy is:',score)\nf1 = metrics.f1_score(y_test, pred)\nprint('F score is:',f1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"808fa0d16596a3fb934c5ae96aa2cadabb6228bd"},"cell_type":"code","source":"sns.heatmap(metrics.confusion_matrix(pred,y_test),annot=True,fmt='2.0f')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf556962795d16e76cccaa0df7e5648c21ebe8e0"},"cell_type":"code","source":"test = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62d51f2cdbe5f9b36d85dbf0b867d9b21aae8e08"},"cell_type":"code","source":"test = test.drop('qid',axis=1)\n\ntest['question_text'] = test.question_text.apply(lambda x: x.lower())\ntest['question_text'] = test.question_text.apply(lambda x: ''.join([c for c in x if c not in punctuation]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e6701aff4329fcd8c1f35a156d9146c8a0a81dce"},"cell_type":"code","source":"test['question_text'] = test.question_text.apply(lambda x: word_tokenize(x))\n\ntest['question_text'] = test.question_text.apply(lambda x: [w for w in x if w not in stop_words])\n\ntest['question_text'] = test.question_text.apply(lambda x: [w for w in x if w not in dup_words])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b09495e6110b3fbcf12c1b0a8399b9a3ae2e819"},"cell_type":"code","source":"test['question_text'] = test.question_text.apply(lambda x: ' '.join(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f62298732d6262ad14ad0c64e4a3d012d25d2438"},"cell_type":"code","source":"# Transform the training data using only the 'text' column values: count_train \ncount = count_vectorizer.transform(test.question_text)\n\nX = ch2.transform(count)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3a12e354da70b6ba1d0e960358955cec72af702"},"cell_type":"code","source":"y_pred = clf.predict(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1121d7f337ce9bafbad25a4ffb355881361653f"},"cell_type":"code","source":"submission = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a33d23a88d999cc4574da8f9050cafdf7781ccf0"},"cell_type":"code","source":"submission['prediction'] = y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b16984cf77e50bd70bd9ee279e8e3d0663e84f82"},"cell_type":"code","source":"submission.to_csv('submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28e6559726df0c37c640b5e1f2c1e3522cc2493a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}