{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Loading dataset"},{"metadata":{},"cell_type":"markdown","source":"Import libraries needed in this notebook"},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd \nimport numpy as np \n%matplotlib inline\nimport matplotlib as mp \nimport matplotlib.pyplot as plt\nimport seaborn as sns\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"here are the first five rows"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_raw = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ndata_raw.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"dataset attributes and their data types"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_raw.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_raw.dropna(inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Number of sincere and insincere questions in this dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_raw.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let's plot their counts"},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(data=data_raw, x='target')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_counts = data_raw.target.value_counts()\nsincere_q_pc = val_counts[0]/val_counts.sum()\nsincere_q_pc = sincere_q_pc*100\nprint('{}% of questions are sincere while the rest are insincere'.format(sincere_q_pc))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"The vast majority of questions in this dataset are sincere and only a small number are insincere "},{"metadata":{},"cell_type":"markdown","source":"Let us take a look at the questions in this dataset and try finding out how they have been classified as sincere and insincere."},{"metadata":{"trusted":true},"cell_type":"code","source":"insincere_questions = data_raw[data_raw['target'] == 1].question_text\nsincere_questions = data_raw[data_raw['target'] == 0].question_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"insincere_questions.sample(3, random_state=1).values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Sweet lord, do these questions look pathetic!"},{"metadata":{},"cell_type":"markdown","source":"These are examples of some of the sincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere_questions.sample(3, random_state=1).values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## EDA and Visualization"},{"metadata":{},"cell_type":"markdown","source":"- Before we start analysing text data, it is important that it is filtered of stop words, punctuations, contractions, and every document of text is tokenized.\n\n- After cleaning up and tokenizing text, we need to convert every word to its root form so analysis and querying of data can be sped up."},{"metadata":{},"cell_type":"markdown","source":"### Cleaning of text data and preprocessing"},{"metadata":{},"cell_type":"markdown","source":"Importing libraries for text clean up"},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\nimport sys\nimport spacy\n\n#nltk.download('stopwords')\n#nltk.download('averaged_perceptron_tagger')\n#nltk.download('wordnet')\n\nfrom nltk.corpus import stopwords\nimport string","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Function for cleaning text"},{"metadata":{"trusted":true},"cell_type":"code","source":"nlp = spacy.load(\"en_core_web_sm\", disable=['parser','ner'])\nstop = set(stopwords.words('english'))\npunc = set(string.punctuation)\n\ndef clean_text(text):\n    # Convert the text into lowercase\n    text = text.lower()\n    # Split into list\n    wordList = text.split()\n    # Remove punctuation\n    wordList = [\"\".join(x for x in word if (x==\"'\")|(x not in punc)) for word in wordList]\n    # Remove stop words\n    wordList = [word for word in wordList if word not in stop]\n\n    reformed_sentence = \" \".join(wordList)\n    doc = nlp(reformed_sentence)\n    return \" \".join([token.lemma_ for token in doc])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Let us take a sample question and run this function on it to see if it works"},{"metadata":{"trusted":true},"cell_type":"code","source":"question = data_raw.question_text.sample(1, random_state=1).values[0]\nquestion","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_text(question)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Nice. Our text clean up function works"},{"metadata":{},"cell_type":"markdown","source":"Now we will clean every row of text data by running this function"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_raw['clean_text'] = data_raw['question_text'].astype('str').apply(clean_text)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualizations"},{"metadata":{},"cell_type":"markdown","source":"Build word clouds of sincere and insincere questions to find the most frequently occuring words in each."},{"metadata":{},"cell_type":"markdown","source":"Import the wordcloud library"},{"metadata":{"trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud, ImageColorGenerator","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"function for splitting sentences into a dictionary of uniquely occuring words and their frequencies"},{"metadata":{"trusted":true},"cell_type":"code","source":"def word_freq_dict(text):\n    # Convert text into word list\n    wordList = text.split()\n    # Generate word freq dictionary\n    wordFreqDict = {word: wordList.count(word) for word in wordList}\n    return wordFreqDict","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"function for plotting a wordcloud from a word frequency dictionary"},{"metadata":{"trusted":true},"cell_type":"code","source":"def word_cloud_from_frequency(word_freq_dict, title, figure_size=(10,6)):\n    wordcloud.generate_from_frequencies(word_freq_dict)\n    plt.figure(figsize=figure_size)\n    plt.imshow(wordcloud)\n    plt.axis(\"off\")\n    plt.title(title)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Wordcloud of a random sample of 1000 insincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# join every question as a single sample of text\ninsincere_questions = data_raw.clean_text[data_raw['target'] == 1]\ninsincere_sample = \" \".join(insincere_questions.sample(1000, random_state=1).values)\ninsincere_word_freq = word_freq_dict(insincere_sample)\nwordcloud = WordCloud(width= 5000,\n    height=3000,\n    max_words=200,\n    colormap='Reds',\n    background_color='white')\n\nword_cloud_from_frequency(insincere_word_freq, \"Most Frequent Words in a sample of 1000 questions flagged insincere\") ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Wordcloud of a random sample of sincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere_questions = data_raw[data_raw['target'] == 0].clean_text\nsincere_sample = \" \".join(sincere_questions.sample(1000, random_state=1).astype('str').values)\nsincere_word_freq = word_freq_dict(sincere_sample)\nwordcloud = WordCloud(width= 5000,\n    height=3000,\n    max_words=200,\n    colormap='Greens',\n    background_color='white')\n\nword_cloud_from_frequency(sincere_word_freq, \"Most Frequent Words in a sample of 1000 questions flagged sincere\") ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Text to vector matrix conversion "},{"metadata":{},"cell_type":"markdown","source":"We need to transform text into a matrix of vectors"},{"metadata":{},"cell_type":"markdown","source":"### Using Bag of words model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nbow_converter = CountVectorizer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_question_text = data_raw['clean_text'].sample(1, random_state= 1).values\nsample_question_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_count_vectorized_data = bow_converter.fit_transform(sample_question_text)\nsample_count_vectorized_data.toarray()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_vectorized_data_feature_names = bow_converter.get_feature_names()\ncount_vectorized_data_feature_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(count_vectorized_data_feature_names)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Using Tfidf(term-frequency-inverse-document-frequency) model"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\ntfidf_converter = TfidfVectorizer(ngram_range=(1,1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_tfidf_vectorized_data = tfidf_converter.fit_transform(sample_question_text)\nsample_tfidf_vectorized_data.toarray()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfidf_word_feature_names = tfidf_converter.get_feature_names()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfidf_word_feature_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(tfidf_word_feature_names)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building a logistic regression model"},{"metadata":{},"cell_type":"markdown","source":"### Pipeline with logistic regression and count vectorizer"},{"metadata":{},"cell_type":"markdown","source":"define count vactorizer and logistic regression models"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.metrics import accuracy_score, confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\ncount_vectorizer = CountVectorizer()\nmodel = LogisticRegression(C=1, random_state=0)\n\nvectorize_logit_pipeline = Pipeline([\n    ('count_vectorizer', count_vectorizer),\n    ('logit', model)\n])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"- define input and target variables\n- split training dataset into train and test sets\n- train the model using the feature and target training sets"},{"metadata":{"trusted":true},"cell_type":"code","source":"# input variable\nX = data_raw['clean_text']\n# target variable\ny = data_raw['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X, test_X, train_y, test_y = train_test_split(X, y, test_size=0.3)\nvectorize_logit_pipeline.fit(train_X, train_y)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Get the predictions from the model"},{"metadata":{"trusted":true},"cell_type":"code","source":"predictions = vectorize_logit_pipeline.predict(test_X)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check the accuracy score"},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy_score(test_y, predictions)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Check the f1 score and plot the confusion matrix"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score\nf1_score(test_y, predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"confusion_matrix_logit_cv = confusion_matrix(test_y, predictions)\nsns.heatmap(confusion_matrix_logit_cv, annot= True, xticklabels=['sincere', 'insincere'], yticklabels=['sincere', 'insincere'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report\nprint(classification_report(test_y, predictions))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define a pipeline with logistic regression and tfidf bi-grams vectorizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"tfidf_ngrams_converter = TfidfVectorizer(ngram_range=(1,2))\ntfidf_ngrams_logit_pipeline = Pipeline([\n    ('tfidf_vectorizer', tfidf_ngrams_converter),\n    ('logit', model)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tfidf_ngrams_logit_pipeline.fit(train_X, train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"new_predictions = tfidf_ngrams_logit_pipeline.predict(test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"accuracy_score(test_y, new_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"f1_score(test_y, new_predictions)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"confusion_matrix_logit_tfidf = confusion_matrix(test_y, new_predictions)\nsns.heatmap(confusion_matrix_logit_tfidf, annot= True, xticklabels=['sincere', 'insincere'], yticklabels=['sincere', 'insincere'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(test_y, new_predictions, target_names=['sincere', 'insincere']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we can observe that the when bigram word vector features are included, our logit model gave us the best accuracy and f1 scores."},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\ntest_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data['clean_text'] = test_data['question_text'].astype('str').apply(clean_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_final = test_data['clean_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_final = tfidf_ngrams_logit_pipeline.predict(X_final)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_final[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data['target'] = y_final","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df = test_data[['qid', 'target']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df.rename(columns={'target': 'prediction'}, inplace=True)\nresult_df.set_index('qid', inplace=True)\nresult_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_df.to_csv('submission.csv')\n!head submission.csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}