{"cells":[{"metadata":{"_uuid":"4a030df26f181aa56a57533712d49ad19d3bed55"},"cell_type":"markdown","source":"# Topic Modelling using Latent Semantic Analysis"},{"metadata":{"_uuid":"fc355d72054822571c1d44f3ec5be7d675fdb4f2"},"cell_type":"markdown","source":"## Import of libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport matplotlib as mp\nimport os\nfrom nltk.corpus import stopwords\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.decomposition import LatentDirichletAllocation,TruncatedSVD\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"449625e22659ebef3f222529c139d1777ff83e56"},"cell_type":"markdown","source":"## Data upload"},{"metadata":{"trusted":true,"_uuid":"89ef655f8e086eecc46e352732c93faa8a2e7147"},"cell_type":"code","source":"#uploading data in dataframe\ntrain=pd.read_csv(\"../input/train.csv\",sep=',')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"31497fa694ad872a081deb1b7465a6acbfae6832"},"cell_type":"code","source":"#displaying exemple data\ntrain.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d62aa2d20fc03388a187b97a151302275fe4d8f9"},"cell_type":"code","source":"#displaying exemple of insincere data \ntrain[train.target==1].head(5)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"#displayin dataframe info\ntrain.info()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6475e60030b621600d091a9a19aaf209fbf92976"},"cell_type":"markdown","source":"There is no missing values "},{"metadata":{"trusted":true,"_uuid":"c2272740cd4397c43c6d23d2ca813d697fd1f869"},"cell_type":"code","source":"#counting target values\ntrain.target.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6238b83edaf590878f6c1724db6ea106481a284b"},"cell_type":"markdown","source":"We have to deal with unbalanced target Feature..."},{"metadata":{"trusted":true,"_uuid":"446a09421825a3538ce5afc326d7bc67becc570a"},"cell_type":"markdown","source":"## Feature extraction"},{"metadata":{"trusted":true,"_uuid":"2f37090c842b4e3a75fbdb592f4fe5bf63298c77"},"cell_type":"code","source":"train['word_count'] = train['question_text'].apply(lambda x: len(str(x).split(\" \")))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b3023b14aa5752370f2df8d03c767cd5251784b3"},"cell_type":"code","source":"#basic statistic about word_count\ntrain.word_count.describe()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"976682c66facb28226d4a7d8663662c7d8a85f2f"},"cell_type":"markdown","source":"## Text transformation"},{"metadata":{"trusted":true,"_uuid":"494070dc43fd53ed59173705aab7b184ecf871e2"},"cell_type":"code","source":"#lower case\ntrain['question_text'] = train['question_text'].apply(lambda x: \" \".join(x.lower() for x in x.split()))\n#Removing Punctuation\ntrain['question_text'] = train['question_text'].str.replace('[^\\w\\s]','')\n#Removing numbers\ntrain['question_text'] = train['question_text'].str.replace('[0-9]','')\n#Remooving stop words and words with length <=2\nfrom nltk.corpus import stopwords\nstop = stopwords.words('english')\ntrain['question_text'] = train['question_text'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop and len(x)>2))\n#Stemming\n#from nltk.stem import SnowballStemmer\n#ss=SnowballStemmer('english')\n#train['question_text'] = train['question_text'].apply(lambda x: \" \".join(ss.stem(x) for x in x.split()))\nfrom nltk.stem import WordNetLemmatizer\nwl = WordNetLemmatizer()\ntrain['question_text'] = train['question_text'].apply(lambda x: \" \".join(wl.lemmatize(x,'v') for x in x.split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce67fb852e48625b9a764c3b4b4fa7052b0fdb03"},"cell_type":"code","source":"from nltk.stem import SnowballStemmer,WordNetLemmatizer,PorterStemmer,LancasterStemmer\nwl = WordNetLemmatizer()\nss=SnowballStemmer('english')\nps=PorterStemmer()\nls=LancasterStemmer()\ntest_list=['does','peaople','writing','beards','enjoyment','bought','leaves','gave','given','generaly','would']\nfor item in test_list :\n    print('lemmatizer : %s'%wl.lemmatize(item,'v'))\n    print('SS stemmer : %s'%ss.stem(item))\n    print('PS stemmer : %s'%ps.stem(item))\n    print('LS stemmer : %s'%ls.stem(item))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"d1844ebb14282c0f777d237c3f67d93b275e6586"},"cell_type":"code","source":"train.head(5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90387c4013ec6ba2d861f55ed66a3e41419c9233"},"cell_type":"markdown","source":"## Topic Modeling insincere questions\nFor topic modeling we are going to use a TFIDF matrix transformation."},{"metadata":{"trusted":true,"_uuid":"eacaff6578a07212423d9ad91f1c0932cc55b5f4"},"cell_type":"code","source":"tfidf_v = TfidfVectorizer(min_df=20,max_df=0.8,sublinear_tf=True,ngram_range={1,2})\n#matrixTFIDF= tfidf_v.fit_transform(train.question_text)\nmatrixTFIDF= tfidf_v.fit_transform(train[train.target==1].question_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b472a304d044e79e1a6178088dd800881116f02"},"cell_type":"code","source":"print(matrixTFIDF.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b969511465ffa81feac6084e6c99b2d306bc4eaf"},"cell_type":"markdown","source":"### Topic modeling using LSA"},{"metadata":{"trusted":true,"_uuid":"af45d25b284527f09dd1822901c7217c3fff8794"},"cell_type":"code","source":"svd=TruncatedSVD(n_components=15, n_iter=10,random_state=42)\nX=svd.fit_transform(matrixTFIDF)             ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ba08750145db0e04dac8330eb39ea1382a07c7e5"},"cell_type":"markdown","source":"get_topics give the n most contributif words in a topic"},{"metadata":{"trusted":true,"_uuid":"dff52ea9b7ece413f3d37634b95e941af309269e"},"cell_type":"code","source":"def get_topics(components, feature_names, n=15):\n    for idx, topic in enumerate(components):\n        print(\"Topic %d:\" % (idx))\n        print([(feature_names[i], topic[i])\n                        for i in topic.argsort()[:-n - 1:-1]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"509f19ec98ba24c1aac6ac1829a2b98e576d0ede"},"cell_type":"code","source":"get_topics(svd.components_,tfidf_v.get_feature_names())","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}