{"cells":[{"metadata":{"trusted":false,"_uuid":"e6f784873ba2c06bc3c8ccb5ae589593c6b33ed4"},"cell_type":"code","source":"import pandas as pd\nimport nltk\nimport string\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f9844ff6a4530bc45d3b5a629247afda436b528d"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\n#test_df = pd.read_csv(\"ProjectData/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f29a0d540fb3c25a0b25007105a2f0c1e78397d8"},"cell_type":"code","source":"train_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a7d8dbc4f805322178b0cb0c100868b8752d78cf"},"cell_type":"code","source":"train_text = train_df['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"197c83d3d3639ba0f8f416ef717d8d3f7a4bae8c"},"cell_type":"code","source":"type(train_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"17b79d6478ac09c3366e59c8c356dbac59e20cb0"},"cell_type":"code","source":"train_label = train_df['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d25774cb38ebe4c7e999eccfabe103b99557c902"},"cell_type":"code","source":"train_label.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22b8949c91458bc9e6995627db580fa5268e9962"},"cell_type":"markdown","source":"**WordCloud for Insincere Questions**"},{"metadata":{"trusted":false,"_uuid":"84cc02d6fe0735f5e21aff7581cac13463f976fc"},"cell_type":"code","source":"from wordcloud import WordCloud","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5494dd5baffe0f7ee861eb48959d6b0f92e7b7b2"},"cell_type":"code","source":"insincere_text = train_df[train_df.target ==1 ]['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"58b905e267a087f6d632c6a635e56c14b2f336fc"},"cell_type":"code","source":"insincere_text = \" \".join(text for text in insincere_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"fd52d6ec08a1d0bdfc40c81f62c6bb0095d0cadb"},"cell_type":"code","source":"insincere_wordcloud = WordCloud(max_font_size=50, max_words=100, background_color=\"white\").generate(insincere_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"f751ac1951a34ce5b8f7751c7bd2d27483ecb0b6"},"cell_type":"code","source":"# Display the generated image:\nplt.figure(figsize=(15,10))\nplt.imshow(insincere_wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f523492b0cd4fab0fb17bf9b634ffafb7566b0fb"},"cell_type":"markdown","source":"**Examples of Insincere questions**"},{"metadata":{"trusted":false,"_uuid":"1c8dd5d4948a9695643214ea8386a772286a5dbd"},"cell_type":"code","source":"insincere_questions_corpus = train_df[train_df.target ==1 ]['question_text'].values.tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"640abe8c067ccd55ddf9660233d733ac12892b06"},"cell_type":"code","source":"insincere_questions_corpus[:11]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"ba5b78bdaf5bf38b69b3ec20af2899491f1b85f2"},"cell_type":"code","source":"from nltk.corpus import stopwords","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d2e6223deef55220681dd1249dd524502c9269ac"},"cell_type":"code","source":"from nltk.tokenize import word_tokenize,sent_tokenize\nfrom nltk.stem import WordNetLemmatizer\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"9392cd126c39451f6939a2940d1a064e674ff845"},"cell_type":"code","source":"def clean_text(insincere_words):\n    #print(insincere_words)\n    insincere_words_tokenized = word_tokenize(insincere_words)\n    #insincere_words_tokenized = [i for i in insincere_words_tokenized if nltk.pos_tag ]\n    insincere_words_cleaned = [i.lower() for i in insincere_words_tokenized]\n    insincere_words_cleaned = [WordNetLemmatizer().lemmatize(i) for i in insincere_words_cleaned]\n    insincere_words_cleaned = [i for i in insincere_words_cleaned if i not in string.punctuation]\n    insincere_words_cleaned = [i for i in insincere_words_cleaned if i not in stopwords.words('english')]\n    return(insincere_words_cleaned)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e75466a688f84e5ac672a3da43898bc69b52432f"},"cell_type":"code","source":"insincere_questions_cleaned =[clean_text(doc) for doc in insincere_questions_corpus]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"40992c32ba53c535749f99374c17b42979ee675f"},"cell_type":"code","source":"insincere_questions_cleaned[:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c43b18ec6f4a8991c1597a599ef59e2f2cd50b95"},"cell_type":"code","source":"insincere_questions_text = \" \".join(str(i) for i in insincere_questions_cleaned)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1476dbe9866fad83494730b4c9591fb0308daf9e"},"cell_type":"markdown","source":"**Another iteration of wordcloud after some text preprocessing**"},{"metadata":{"trusted":false,"_uuid":"52d3dacfcada22899129fc50927115f59bb3033b"},"cell_type":"code","source":"insincere_wordcloud = WordCloud(max_font_size=50, max_words=100, background_color=\"white\").generate(insincere_questions_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"61d6924d1d37fceac01a2a75d73fa9e034ef9eda"},"cell_type":"code","source":"# Display the generated image:\nplt.figure(figsize=(15,10))\nplt.imshow(insincere_wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"60fca70b1bbfbe236e80db2937340477396b909d"},"cell_type":"markdown","source":"**Topic Modeling using LDA**"},{"metadata":{"trusted":false,"_uuid":"aceb953ca72fdcb67f9a8f997d488a3192daed82"},"cell_type":"code","source":"import gensim","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b6fc8f40e8d07cbc3ca09120e42ea57d7ec19cd5"},"cell_type":"code","source":"from gensim.models import LdaModel","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a39c4a97fa53785b4cb54e8531f319563f6c390b"},"cell_type":"code","source":"from gensim.corpora import Dictionary","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"3368e59d87bdfaf72e0cbe71b254e1dab2bed43c"},"cell_type":"code","source":"dictionary = Dictionary(insincere_questions_cleaned)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"d353cc475afff371fa0facb1063223e64ee8b729"},"cell_type":"code","source":"len(dictionary)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cf93ca00c6f0419e6f77e36e4af5939efcfccc44"},"cell_type":"code","source":"insincere_bow = [dictionary.doc2bow(doc) for doc in insincere_questions_cleaned]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8680c19ed7ae5947b785859cd730cda3d5be0992"},"cell_type":"code","source":"from gensim.models.ldamodel import LdaModel","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"47ac67e3730da53e6d77bfe585084e28448e3e6f"},"cell_type":"code","source":"lda_model = LdaModel(corpus = insincere_bow, num_topics=5, id2word=dictionary, passes = 10,random_state = 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"88e158937adcc285018793f46ed3d20e1e93ca97"},"cell_type":"code","source":"lda_model.show_topics(num_topics= 5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"911f1d70e757c8e6fc199ed9b01acb16ec802e97"},"cell_type":"markdown","source":"Following topics seem to generate most insincere questions:\n* America and American politics\n* Religion \n* Gender and Race"},{"metadata":{"trusted":false,"_uuid":"17b52580bbc0987e73a8e360103434678a9112fe"},"cell_type":"code","source":"import pyLDAvis.gensim","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"e26cd6a85f245d11d81e8cd6a246009db4d3c5f8"},"cell_type":"code","source":"lda_visualization = pyLDAvis.gensim.prepare(lda_model,insincere_bow,dictionary,sort_topics = False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"1cc539f48671ada2b673d1b30c4defbc6d37a78f"},"cell_type":"code","source":"pyLDAvis.display(lda_visualization)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}