{"cells":[{"metadata":{"_uuid":"4a030df26f181aa56a57533712d49ad19d3bed55"},"cell_type":"markdown","source":"# Quora data exploration (EDA) and Topic modeling (LSA vs LDA)"},{"metadata":{"_uuid":"fc355d72054822571c1d44f3ec5be7d675fdb4f2"},"cell_type":"markdown","source":"## Import of libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib as mp\nimport os\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import LatentDirichletAllocation,TruncatedSVD\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"449625e22659ebef3f222529c139d1777ff83e56"},"cell_type":"markdown","source":"## Data upload"},{"metadata":{"trusted":true,"_uuid":"89ef655f8e086eecc46e352732c93faa8a2e7147"},"cell_type":"code","source":"#uploading data in dataframe\ntrain=pd.read_csv(\"../input/train.csv\",sep=',')\ntest=pd.read_csv(\"../input/test.csv\",sep=',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bd39b53e1042d6260038f49efc010d09cb3bce3"},"cell_type":"code","source":"#displayin shapes\nprint ('train shapes : %s'%str(train.shape))\nprint ('test shapes : %s'%str(test.shape))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31497fa694ad872a081deb1b7465a6acbfae6832"},"cell_type":"code","source":"#displaying exemple data\ntrain.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d62aa2d20fc03388a187b97a151302275fe4d8f9"},"cell_type":"code","source":"#displaying exemple of insincere data \ntrain[train.target==1].head(5)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"#displayin dataframe info\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6475e60030b621600d091a9a19aaf209fbf92976"},"cell_type":"markdown","source":"There is no missing values "},{"metadata":{"trusted":true,"_uuid":"c2272740cd4397c43c6d23d2ca813d697fd1f869"},"cell_type":"code","source":"#counting target values\ntrain.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6238b83edaf590878f6c1724db6ea106481a284b"},"cell_type":"markdown","source":"We have to deal with unbalanced target Feature..."},{"metadata":{"trusted":true,"_uuid":"446a09421825a3538ce5afc326d7bc67becc570a"},"cell_type":"markdown","source":"## Feature extraction"},{"metadata":{"_uuid":"8938a8cfd54fc6d9ba4d6dc61dfbf081cebb50e4"},"cell_type":"markdown","source":"I tried first to make prediction with only extracted features but the result wasn't good"},{"metadata":{"trusted":true,"_uuid":"2f37090c842b4e3a75fbdb592f4fe5bf63298c77"},"cell_type":"code","source":"train['word_count'] = train['question_text'].apply(lambda x: len(str(x).split(\" \")))\n#train['char_count'] = train['question_text'].str.len()\n#stop = stopwords.words('english')\n#train['stopwords'] = train['question_text'].apply(lambda x: len([x for x in x.split() if x in stop]))\n#train['numerics'] = train['question_text'].apply(lambda x: len([x for x in x.split() if x.isdigit()]))\n#train['upper'] = train['question_text'].apply(lambda x: len([x for x in x.split() if x.isupper()]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3023b14aa5752370f2df8d03c767cd5251784b3"},"cell_type":"code","source":"#basic statistic about word_count\ntrain.word_count.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"e83ce9dbfff6f6ab9fbd8abd3d1de16645cd6447"},"cell_type":"code","source":"#ploting box plot of word_count by target without outlier\ntrain.boxplot(column='word_count', by='target', grid=False,showfliers=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"189497a87f9c4279c08e92b84d94473e39ed6669"},"cell_type":"markdown","source":"As we can see, Quora's questions is composed from few word (mainly <25 words ). the distribution for in insincere question is more spread out."},{"metadata":{"_uuid":"976682c66facb28226d4a7d8663662c7d8a85f2f"},"cell_type":"markdown","source":"## Text transformation"},{"metadata":{"trusted":true,"_uuid":"494070dc43fd53ed59173705aab7b184ecf871e2"},"cell_type":"code","source":"#lower case\ntrain['question_text'] = train['question_text'].apply(lambda x: \" \".join(x.lower() for x in x.split()))\n#Removing Punctuation\ntrain['question_text'] = train['question_text'].str.replace('[^\\w\\s]','')\n#Removing numbers\ntrain['question_text'] = train['question_text'].str.replace('[0-9]','')\n#Remooving stop words and words with length <=2\nfrom nltk.corpus import stopwords\nstop = stopwords.words('english')\ntrain['question_text'] = train['question_text'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop and len(x)>2))\n#Stemming\n#from nltk.stem import SnowballStemmer\n#ss=SnowballStemmer('english')\n#train['question_text'] = train['question_text'].apply(lambda x: \" \".join(ss.stem(x) for x in x.split()))\nfrom nltk.stem import WordNetLemmatizer\nwl = WordNetLemmatizer()\ntrain['question_text'] = train['question_text'].apply(lambda x: \" \".join(wl.lemmatize(x,'v') for x in x.split()))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fb9cd2e631542a7e0059b2247eccfd6f84b8b971"},"cell_type":"markdown","source":"I tested several words on nltk stemmer et lemmatizer and i choose to use snowball stemmer"},{"metadata":{"trusted":true,"_uuid":"ce67fb852e48625b9a764c3b4b4fa7052b0fdb03"},"cell_type":"code","source":"from nltk.stem import SnowballStemmer,WordNetLemmatizer,PorterStemmer,LancasterStemmer\nwl = WordNetLemmatizer()\nss=SnowballStemmer('english')\nps=PorterStemmer()\nls=LancasterStemmer()\ntest_list=['does','peaople','writing','beards','enjoyment','bought','leaves','gave','given','generaly','would']\nfor item in test_list :\n    print('lemmatizer : %s'%wl.lemmatize(item,'v'))\n    print('SS stemmer : %s'%ss.stem(item))\n    print('PS stemmer : %s'%ps.stem(item))\n    print('LS stemmer : %s'%ls.stem(item))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"d1844ebb14282c0f777d237c3f67d93b275e6586"},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a02797b7018fa7f7531daa4a159c476e59805926"},"cell_type":"markdown","source":"## Most frequent terms for sincere and insincere questions\n\nget_words_freq return for a corpus the sorted list of words by frequency  "},{"metadata":{"trusted":true,"_uuid":"525660d67bb8823d783c32803b28e9e7658e8bd3"},"cell_type":"code","source":"def get_words_freq(corpus):\n    vec = CountVectorizer(ngram_range={1,2}).fit(corpus)\n    #bag of words its a sparse document item matrix\n    bag_of_words = vec.transform(corpus)\n    #we calculate the occurrence for each term. warning, the sum of matrix is a 1 row matrix\n    sum_words = bag_of_words.sum(axis=0) \n    # Vocabulary_ its a dictionary { word :position }  \n    words_freq = [(word, sum_words[0, idx]) for word, idx in     vec.vocabulary_.items()]\n    words_freq =sorted(words_freq, key = lambda x: x[1],reverse=True)\n    return words_freq","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e60d55638510755f875ceea6a57b1c2752155bd1"},"cell_type":"markdown","source":"Let' see the 30 most frequent terms of sincere question :"},{"metadata":{"trusted":true,"_uuid":"a0f58f30ae86632e1802d325929626dee5537276"},"cell_type":"code","source":"top_sincere=get_words_freq(train[train.target==0].question_text)\nprint(top_sincere[:30])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40799d90cd603dcdf5e160ce1c6f861c7b1261de"},"cell_type":"markdown","source":"Let' see the 30 most frequent terms of insincere question :"},{"metadata":{"trusted":true,"_uuid":"6a4f196c60e8773a6b208cee967f3e6765d4c1f6","scrolled":false},"cell_type":"code","source":"top_insincere=get_words_freq(train[train.target==1].question_text)\nprint(top_insincere[:30])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f739f8e4554783841538c0001da965332c4f5e29"},"cell_type":"code","source":"#[y[0] for y in top_sincere].index('black people')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"881d5a5e6c72cc946aae0dcb9fd1deb7dc4293dc"},"cell_type":"markdown","source":"We can also use wordcloud to visualize the most frequent terms for insincere questions"},{"metadata":{"trusted":true,"_uuid":"a70c9ea2e2fb2b6fa55eb1f0578a7a07810f842e"},"cell_type":"code","source":"from wordcloud import WordCloud\nwc=WordCloud(background_color='white')\nwc.generate(''.join(train[train.target==1].question_text))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"011c985a92cfede68cb5fdab0a78bd6e41a24e00","scrolled":true},"cell_type":"code","source":"#let's plot\nplt.figure(1, figsize=(15, 15))\nplt.axis('off')\nplt.imshow(wc)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90387c4013ec6ba2d861f55ed66a3e41419c9233"},"cell_type":"markdown","source":"## Topic Modeling insincere questions\nFor topic modeling we are going to use a TFIDF matrix transformation."},{"metadata":{"trusted":true,"_uuid":"eacaff6578a07212423d9ad91f1c0932cc55b5f4"},"cell_type":"code","source":"tfidf_v = TfidfVectorizer(min_df=20,max_df=0.8,sublinear_tf=True,ngram_range={1,2})\n#matrixTFIDF= tfidf_v.fit_transform(train.question_text)\nmatrixTFIDF= tfidf_v.fit_transform(train[train.target==1].question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b472a304d044e79e1a6178088dd800881116f02"},"cell_type":"code","source":"print(matrixTFIDF.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63f0aa81012a677a1d1101e26d9868bdc6b511e8"},"cell_type":"code","source":"plt.boxplot(np.array(matrixTFIDF.mean(axis=0).transpose()),showfliers=False)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b969511465ffa81feac6084e6c99b2d306bc4eaf"},"cell_type":"markdown","source":"### Topic modeling using LSA"},{"metadata":{"trusted":true,"_uuid":"af45d25b284527f09dd1822901c7217c3fff8794"},"cell_type":"code","source":"svd=TruncatedSVD(n_components=15, n_iter=10,random_state=42)\nX=svd.fit_transform(matrixTFIDF)             ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f6404d30083fa13bbfd350e7fb54d498576c591"},"cell_type":"code","source":"plt.plot(svd.singular_values_[0:15])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8c5625cc60d5c899c10f5643a7ab1ad456a2530","scrolled":true},"cell_type":"code","source":"#Explained variance by our components\nnp.sum(svd.explained_variance_ratio_[0:15])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72235be8ceedbce45097d3e70973c5c1bffee05e"},"cell_type":"markdown","source":"5%! it's very low..."},{"metadata":{"trusted":true,"_uuid":"0c0378c2973a56edad3cd91da483aefce013550e"},"cell_type":"code","source":"#components_ give the word contribution for each component \nsvd.components_.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba08750145db0e04dac8330eb39ea1382a07c7e5"},"cell_type":"markdown","source":"get_topics give the n most contributif words in a topic"},{"metadata":{"trusted":true,"_uuid":"dff52ea9b7ece413f3d37634b95e941af309269e"},"cell_type":"code","source":"def get_topics(components, feature_names, n=15):\n    for idx, topic in enumerate(components):\n        print(\"Topic %d:\" % (idx))\n        print([(feature_names[i], topic[i])\n                        for i in topic.argsort()[:-n - 1:-1]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"509f19ec98ba24c1aac6ac1829a2b98e576d0ede"},"cell_type":"code","source":"get_topics(svd.components_,tfidf_v.get_feature_names())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75f80f61348deffb132736d454928a3f666e7eb5"},"cell_type":"markdown","source":"### Topic modeling using LDA"},{"metadata":{"trusted":true,"_uuid":"9bc21ae6afc1b8860c80f17525687f7d563da615"},"cell_type":"code","source":"lda=LatentDirichletAllocation(n_components=15,random_state=42,max_iter=10)\nZ=lda.fit_transform(matrixTFIDF)  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d58fcf4a2f1ced9ffe478349099054ccf930fb78"},"cell_type":"code","source":"get_topics(lda.components_,tfidf_v.get_feature_names(),n=15)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}