{"cells":[{"metadata":{"trusted":true,"_uuid":"a11af005b960e412c508937426671fe210c9a26f","_kg_hide-input":false},"cell_type":"code","source":"# import all needed packages\nimport re\nimport os\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport wordcloud as wc\nimport matplotlib.pyplot as plt\nfrom collections import Counter\nfrom afinn import Afinn\nfrom nltk.corpus import sentiwordnet as swn\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca9f0fb5a9afedc388dd5397ad83f4c3acf35fb8"},"cell_type":"code","source":"# get the training data\n# downloaded from https://www.kaggle.com/c/quora-insincere-questions-classification/data\n# since this is merely a shot exploratory look at the data, the test data is not needed\n# and will therefore not be loaded\ndata = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\", sep = ',')\n\n# drop id column\ndata = data.drop('qid', axis = 1)\n\n# check for missing values\n# none here\ndata.isna().sum()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"75d5ede2a2f5bb28c39ae6ce50a6bb5a47a7bcca"},"cell_type":"code","source":"## the following is used to clean up the data and prepare it for further analysis\n# remove useless special characters\nreplaceing = ['\"', '.', ',', '?', '!']\nfor word in replaceing:\n    data['question_text'] = data['question_text'].str.replace(word, '')\n    \n# expand contractions\n# dict to expand contractions\n# taken from https://gist.github.com/nealrs/96342d8231b75cf4bb82\ncList = {\n  \"ain't\": \"am not\",\n  \"aren't\": \"are not\",\n  \"can't\": \"cannot\",\n  \"can't've\": \"cannot have\",\n  \"'cause\": \"because\",\n  \"could've\": \"could have\",\n  \"couldn't\": \"could not\",\n  \"couldn't've\": \"could not have\",\n  \"didn't\": \"did not\",\n  \"doesn't\": \"does not\",\n  \"don't\": \"do not\",\n  \"hadn't\": \"had not\",\n  \"hadn't've\": \"had not have\",\n  \"hasn't\": \"has not\",\n  \"haven't\": \"have not\",\n  \"he'd\": \"he would\",\n  \"he'd've\": \"he would have\",\n  \"he'll\": \"he will\",\n  \"he'll've\": \"he will have\",\n  \"he's\": \"he is\",\n  \"how'd\": \"how did\",\n  \"how'd'y\": \"how do you\",\n  \"how'll\": \"how will\",\n  \"how's\": \"how is\",\n  \"I'd\": \"I would\",\n  \"I'd've\": \"I would have\",\n  \"I'll\": \"I will\",\n  \"I'll've\": \"I will have\",\n  \"I'm\": \"I am\",\n  \"I've\": \"I have\",\n  \"isn't\": \"is not\",\n  \"it'd\": \"it had\",\n  \"it'd've\": \"it would have\",\n  \"it'll\": \"it will\",\n  \"it'll've\": \"it will have\",\n  \"it's\": \"it is\",\n  \"let's\": \"let us\",\n  \"ma'am\": \"madam\",\n  \"mayn't\": \"may not\",\n  \"might've\": \"might have\",\n  \"mightn't\": \"might not\",\n  \"mightn't've\": \"might not have\",\n  \"must've\": \"must have\",\n  \"mustn't\": \"must not\",\n  \"mustn't've\": \"must not have\",\n  \"needn't\": \"need not\",\n  \"needn't've\": \"need not have\",\n  \"o'clock\": \"of the clock\",\n  \"oughtn't\": \"ought not\",\n  \"oughtn't've\": \"ought not have\",\n  \"shan't\": \"shall not\",\n  \"sha'n't\": \"shall not\",\n  \"shan't've\": \"shall not have\",\n  \"she'd\": \"she would\",\n  \"she'd've\": \"she would have\",\n  \"she'll\": \"she will\",\n  \"she'll've\": \"she will have\",\n  \"she's\": \"she is\",\n  \"should've\": \"should have\",\n  \"shouldn't\": \"should not\",\n  \"shouldn't've\": \"should not have\",\n  \"so've\": \"so have\",\n  \"so's\": \"so is\",\n  \"that'd\": \"that would\",\n  \"that'd've\": \"that would have\",\n  \"that's\": \"that is\",\n  \"there'd\": \"there had\",\n  \"there'd've\": \"there would have\",\n  \"there's\": \"there is\",\n  \"they'd\": \"they would\",\n  \"they'd've\": \"they would have\",\n  \"they'll\": \"they will\",\n  \"they'll've\": \"they will have\",\n  \"they're\": \"they are\",\n  \"they've\": \"they have\",\n  \"to've\": \"to have\",\n  \"wasn't\": \"was not\",\n  \"we'd\": \"we had\",\n  \"we'd've\": \"we would have\",\n  \"we'll\": \"we will\",\n  \"we'll've\": \"we will have\",\n  \"we're\": \"we are\",\n  \"we've\": \"we have\",\n  \"weren't\": \"were not\",\n  \"what'll\": \"what will\",\n  \"what'll've\": \"what will have\",\n  \"what're\": \"what are\",\n  \"what's\": \"what is\",\n  \"what've\": \"what have\",\n  \"when's\": \"when is\",\n  \"when've\": \"when have\",\n  \"where'd\": \"where did\",\n  \"where's\": \"where is\",\n  \"where've\": \"where have\",\n  \"who'll\": \"who will\",\n  \"who'll've\": \"who will have\",\n  \"who's\": \"who is\",\n  \"who've\": \"who have\",\n  \"why's\": \"why is\",\n  \"why've\": \"why have\",\n  \"will've\": \"will have\",\n  \"won't\": \"will not\",\n  \"won't've\": \"will not have\",\n  \"would've\": \"would have\",\n  \"wouldn't\": \"would not\",\n  \"wouldn't've\": \"would not have\",\n  \"y'all\": \"you all\",\n  \"y'alls\": \"you alls\",\n  \"y'all'd\": \"you all would\",\n  \"y'all'd've\": \"you all would have\",\n  \"y'all're\": \"you all are\",\n  \"y'all've\": \"you all have\",\n  \"you'd\": \"you had\",\n  \"you'd've\": \"you would have\",\n  \"you'll\": \"you you will\",\n  \"you'll've\": \"you you will have\",\n  \"you're\": \"you are\",\n  \"you've\": \"you have\"\n}\n\n# replace ´ with ' to get every contraction expanded\ndata[\"question_text\"] = data[\"question_text\"].str.replace('’', \"'\")\n\nc_re = re.compile('(%s)' % '|'.join(cList.keys()))\n\ndef expand_contractions(text, c_re = c_re):\n    def replace(match):\n        return cList[match.group(0)]\n    return c_re.sub(replace, text.lower())\n\n# call the function on the data\ndata['question_text'] = data[\"question_text\"].apply(lambda x: expand_contractions(x))\n\n# make everything lower case text\ndata['question_text'] = data['question_text'].str.lower()\n\n# split into words\ndata['question_text'] = data['question_text'].str.split(' ')\n\n# word count\n# calculate now because stop words might be relevant here\ndata['word_count'] = data['question_text'].str.len()\n\n# remove stopwords\nstop_words = set(stopwords.words('english'))\ndata['question_text'] = data['question_text'].apply(lambda x: [w for w in x if not w in stop_words])\n\n# if question is now empty, add string to symbolize NA\ndata['question_text'] = data['question_text'].apply(lambda x: x if x else ['_NA_'])\n\n# lemmatize the words\nlemma = WordNetLemmatizer()\ndata['question_text'] = data['question_text'].apply(lambda x: [lemma.lemmatize(word, pos = 'v') for word in x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"780803c102068afa9a25d792819d9a6aa8ba704d"},"cell_type":"code","source":"## perform some sentiment analysis using different unsupervised methods\n\n# function to calculate sentinent of sentence with sentiwordnet  \ndef get_senti_score_sentence(sentence, score_type):\n    # function to calculate sentinent of word with sentiwordnet\n    def get_senti_score_word(word, score_type):\n        try:\n            word = list(swn.senti_synsets(word, 'n'))[0]\n        except IndexError:\n            return 999\n        if score_type == 'pos':\n            return word.pos_score()\n        elif score_type == 'neg':\n            return word.neg_score()\n        else:\n            return word.obj_score()\n        \n    if sentence == []:\n        raise ValueError('Empty Sentences not allowed!')\n    score = 0\n    n = len(sentence)\n    for word in sentence:\n        if get_senti_score_word(word, score_type) == 999:\n            n = n - 1\n        else:\n            score += get_senti_score_word(word, score_type)\n    try:\n        avg = score / n\n        return avg\n    except ZeroDivisionError:\n        return np.nan\n\n# apply function to data\ndata['sent_pos'] = data['question_text'].apply(lambda x: get_senti_score_sentence(x, 'pos'))\ndata['sent_neg'] = data['question_text'].apply(lambda x: get_senti_score_sentence(x, 'neg'))\n\n# fill NA Values with 0\ndata['sent_pos'].fillna(0, inplace = True)\ndata['sent_neg'].fillna(0, inplace = True)\n\n# make it a string again to apply affinity analyis\ndata['question_text'] = data['question_text'].apply(lambda x: ' '.join(x))\n\n## map out simple afinnity scores\nafn = Afinn(emoticons = True)\ndata['afinn_score'] = data['question_text'].apply(lambda x: afn.score(x))\n\n## vader sentiment scores\n# because vader is highly correlated with afinn score (and also very slow)\n# it won't be used here any further\n#from nltk.sentiment.vader import SentimentIntensityAnalyzer\n\n# function to get vader compound score\n#analyzer = SentimentIntensityAnalyzer()\n\n#def get_vader_compound_score(sentence, analyzer):\n#    scores = analyzer.polarity_scores(sentence)\n#    return scores['compound']\n\n#data['vader'] = data['question_text'].apply(lambda x: get_vader_compound_score(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f8a4bd8571f375d2d1418de94768c7f10c16e0d1"},"cell_type":"code","source":"## take a sneak peak at the data\ndata[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6ca085530d0637e2766b4c2ce0f17bbd7fdf7db2"},"cell_type":"code","source":"## graphical analysis\n\n# first lets plot the frequency percentages of the target variable\ndata['target'].value_counts().plot(kind = 'pie', labels = ['real', 'insincere'],\n     startangle = 90, autopct = '%1.0f%%')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cdcf6fe687cc1e8597f406befab69a5ccd2d5066"},"cell_type":"code","source":"# as you can see it's clearly not equally balanced\n# accuracy won't be the best metric to evaluate your classifiers in this case\n\n# now lets look at the distributions of the different calculated variables\n# first we will take a look at the afinnity scores\nsns.boxplot(x = 'target', y = 'afinn_score', data = data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a588d651f62b727edc77ab548a2f195b179b6954"},"cell_type":"code","source":"# The afinnity score seems to be lower in insincere questions\n# Since these scores have a range from -4 (being a word with negative connotations) and +4 (the opposite),\n# it seems that people writing insincere questions use more negatively connotated words.\n# However, the variance also seems to be a lot bigger in the group of insincere questions.\n\n# let's see if there is a similar pattern in the sentiwordnet scores\nsns.boxplot(x = 'target', y = 'sent_pos', data = data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9804b9cae2759ee7608caa919fbc4c09dace9bb7"},"cell_type":"code","source":"# Overall the average positive sentinent of a sentence is close to zero in both groups\n# However, the mean of the insincere questions scores is slightly higher (which is rather unexpected)\n# The difference is very small, but with a sample of this size statistically significant\n\n# The negative sentinent scores offer a similar picture\nsns.boxplot(x = 'target', y = 'sent_neg', data = data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"148737458c97662400a76b3c60ff4188e9ac9f3b"},"cell_type":"code","source":"# Generally these plots indicate that the insincere questions are more polarizing in general,\n# both in the negative and in the positive direction. Normal questions are expectedly more neutral.\n\n# Let's also take a look at the distribution of the number of words in a sentence\nsns.boxplot(x = 'target', y = 'word_count', data = data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fc790301529b18958af9a3f4eedcf40968e83634"},"cell_type":"code","source":"# This plot shows that the sentences of insincere questions is on average about 5 words longer than\n# the normal questions. This might be a usefull variable to add if you're going to use shallow learning\n\n# Now let's look at the actual frequency of the words.\n# For starters we'll plot a wordcloud for each group\n\n# split by target variable\nnormal = data.loc[data['target'] == 0]\ntroll = data.loc[data['target'] == 1]\n \n# make it one big string\nnormal = ' '.join(question for question in normal['question_text'].astype(str))\ntroll = ' '.join(question for question in troll['question_text'].astype(str))\n\n# We will start with the normal questions\n# call the wordcloud function\nwordcloud = wc.WordCloud(max_font_size = 160, max_words = 150, \n                         background_color = 'white', width = 800,\n                         height = 500).generate(normal)\n# plot it\nplt.figure(figsize = (15, 10))\nplt.imshow(wordcloud, interpolation = 'bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ceb487b06059ea6b7e9a5bf61b56405b6a679b8c"},"cell_type":"code","source":"# Words like work, help, book, make, use etc. are all used very frequently in the normal questions.\n# They are mostly neutral and indicate someone actually looking for advice or similar things.\n\n# Now let's contrast this with the most frequent words in the insincere questions\nwordcloud2 = wc.WordCloud(max_font_size = 160, max_words = 150,\n                          background_color = 'white', width = 800,\n                          height = 500).generate(troll)\n \n# plot it\nplt.figure(figsize = (15, 10))\nplt.imshow(wordcloud2, interpolation = 'bilinear')\nplt.axis(\"off\")\nplt.show()\n\n# you can also save these images if you want to using the following two lines of code\n# wordcloud.to_file(\"Normal_Wordcloud.png\")\n# wordcloud2.to_file(\"Troll_Wordcloud.png\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6905b120a7a72d7021863de62237b1ec18fb94f8"},"cell_type":"code","source":"# The difference is very obvious. Instead of the generic advice topics (like relationships, work, educatione etc.),\n# the most frequently used words are all very political (for example the frequent use of donald trumps name).\n# This also shows why a simple sentiment analysis of words is not sufficient in this case. The words in general are mostly\n# not negative. The word people for example is neither positive or negative. But coupled with other words it \n# should be possible to perform useful topic analysis\n\n# Wordclouds are a great tool to get a first impression of the word frequencies, but without looking at the\n# actual values as well they might be misleading. The max_font_size parameter for example might cover up some\n# valuable information.\n\n# For that reason, let's look at the actual frequencies of the 15 most common words.","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8e4b049dc38c5d21e8fda31acf6f53481c45ca11"},"cell_type":"code","source":"# convert huge strings to lists\nnormal = normal.split(' ')\ntroll = troll.split(' ')\n\n# get the 15 most common words and their frequency values of each group using Counter\nnormal_count = Counter(normal).most_common(15)\ntroll_count = Counter(troll).most_common(15)\n\n# function to generate input for plotting\ndef get_words_and_values(counter):\n    words = []\n    values = []\n    for i in range(0, len(counter)):\n        words.append(counter[i][0])\n        values.append(counter[i][1])\n    return words, values\n\n# simple plot function\ndef freq_plot(words, values, tit = ''):  \n    plt.bar(words, values)\n    plt.ylabel('Absolute Frequency')\n    plt.title(tit)\n    plt.xticks(rotation = 45)\n    \n# call the functions for normal questions first\nwords, values = get_words_and_values(normal_count)\nfreq_plot(words, values, tit = 'Frequency of the 15 most common words in normal questions')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"adb2dbda3989662b8af72e34775b8ef5e6646c70"},"cell_type":"code","source":"# And now we'll use the same functions to generate a similar image for the insincere questions\nwords, values = get_words_and_values(troll_count)\nfreq_plot(words, values, tit = 'Frequency of the 15 most common words in troll questions')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6b890b336997353a5cece8fc3db4d7b4ee1d652e"},"cell_type":"code","source":"# Apart from the different words it is also noteworthy that the word 'people' has a relatively\n# very high percentage of occurence. This might be usefull knowledge for further modelling\n\n# Since there aren't equal numbers of insincere and normal questions, the absolute frequencies might not\n# be the best metric to compare the two groups.\n\n# Instead, we will now plot the relative frequencies of the 15 most frequent words in each group\n# and compare these using grouped bar plots.","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f0e2fbe7c640556d68e104ba5415831a2ce14140"},"cell_type":"code","source":"# function to calculate the relative frequency of a word\ndef relative_frequency(lst, element):\n    return lst.count(element) / float(len(lst))\n\n# function to get lists for plotting\ndef get_rel_troll_and_normal(words, document):\n    rel_troll = []\n    rel_normal = []\n    for word in words:\n        freq = relative_frequency(troll, word)\n        rel_troll.append(freq)\n        freq = relative_frequency(normal, word)\n        rel_normal.append(freq)\n    return rel_troll, rel_normal    \n\n# simple plot function for grouped bar charts\ndef subcategorybar(X, vals, width = 0.8, ylab = '', tit = '', labs = []):\n    n = len(vals)\n    _X = np.arange(len(X))\n    for i in range(n):\n        plt.bar(_X - width / 2. + i / float(n )* width, vals[i], \n                width = width / float(n), align = 'edge')   \n    plt.xticks(_X, X, rotation = 90)\n    plt.ylabel(ylab)\n    plt.title(tit)\n    plt.legend(labs)\n    \n# Let's start with the 15 most frequent normal words again\nwords, values = get_words_and_values(normal_count)\n\nrel_troll, rel_normal = get_rel_troll_and_normal(words, normal)\n\nsubcategorybar(words, [rel_normal, rel_troll], ylab = 'Relative Frequency',\n               tit = 'Relative Frequency of most common words in normal questions',\n               labs = ['normal', 'troll'])  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2dcb2c3c6ad6856647041ee059367847bd9a79a2"},"cell_type":"code","source":"# and for the insincere questions\nwords, values = get_words_and_values(troll_count)\nrel_troll, rel_normal = get_rel_troll_and_normal(words, troll)\n\nsubcategorybar(words, [rel_troll, rel_normal], ylab = 'Relative Frequency',\n               tit = 'Relative Frequency of most common words in troll questions',\n               labs = ['troll', 'normal'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bb33e1b7b69d338f6d2bc83d47c17034ac8a1b32"},"cell_type":"code","source":"# Some differences are clearly obvious. If you want to use shallow learning and plan to use n-grams for that,\n# you might also want to make these kind of plots for the most frequent n-grams. The code should work for that as well.\n\n# Overall the preceeding exploratory analysis shows some interesting patterns in the data.\n# I hope it was usefull to some of you guys. If you have further suggestions or advice, let me know.","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}