{"cells":[{"metadata":{"_uuid":"d1fec9c31f47607b0e9530447327ec509310e6d6"},"cell_type":"markdown","source":"# Topic Modeling with Gensim\nIn this notebook we will extract topics from our collection of questions using Latent Dirichlet Allocation (LDA) and the Gensim package.\n\nGensim markets itself as \"topic modelling for humans\" and its really fast.\n\nAccording to [NLP for Hackers](https://nlpforhackers.io/topic-modeling/) topic modeling is:\n - Dimensinality Reduction - We reduce dimensionality by representing a text in its topic space instead of its word space.\n - Unsupervised Learning - Topic modeling is similar to clustering.\n - A Form of Tagging - Topic modeling applys multiple tags to a text. (Similar to the tags applied to this kernel above!)\n \n Topic modeling is useful for many situations, including our task of text classification.\n \nFrom the [gensim documentation](https://radimrehurek.com/gensim/tut2.html#transformation-interface) Latent Semantic Indexing (LSI) is a form of dimensionality reduction where documents are transformed into a latent space of lower dimensionality.\n\nLDA is a probabilistic extension of LSA (aka multinomial PCA).  LDA’s topics can be interpreted as probability distributions over words. These distributions are, just like with LSA, inferred automatically from a training corpus. Documents are in turn interpreted as a (soft) mixture of these topics (again, just like with LSA)."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# import packages\nimport numpy as np\nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport spacy\nimport nltk\nimport re\n\nfrom gensim import corpora, models, similarities\nimport pyLDAvis.gensim\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom sklearn.metrics import classification_report\nfrom sklearn.metrics import confusion_matrix\n\nnp.random.seed(27)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cad72c35f00a7fd19af1f893f3f07de30d1ab665"},"cell_type":"code","source":"# setting up default plotting parameters\n%matplotlib inline\n\nplt.rcParams['figure.figsize'] = [15.0, 7.0]\nplt.rcParams.update({'font.size': 22,})\n\nsns.set_palette('Set2')\nsns.set_style('white')\nsns.set_context('talk', font_scale=0.8)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"# suppress warnings\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9a8e39c484e93a93b42718a82247ee3903b4898"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a61184af866d976ed9f5950069a9a532d1014e7"},"cell_type":"code","source":"contractions = {\n\"ain't\": \"is not\",\n\"aren't\": \"are not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he will\",\n\"he'll've\": \"he he will have\",\n\"he's\": \"he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how is\",\n\"I'd\": \"I would\",\n\"I'd've\": \"I would have\",\n\"I'll\": \"I will\",\n\"I'll've\": \"I will have\",\n\"I'm\": \"I am\",\n\"I've\": \"I have\",\n\"i'd\": \"i would\",\n\"i'd've\": \"i would have\",\n\"i'll\": \"i will\",\n\"i'll've\": \"i will have\",\n\"i'm\": \"i am\",\n\"i've\": \"i have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it will\",\n\"it'll've\": \"it will have\",\n\"it's\": \"it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she will\",\n\"she'll've\": \"she will have\",\n\"she's\": \"she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so as\",\n\"that'd\": \"that would\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that is\",\n\"there'd\": \"there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there is\",\n\"they'd\": \"they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they will\",\n\"they'll've\": \"they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\"we'd\": \"we would\",\n\"we'd've\": \"we would have\",\n\"we'll\": \"we will\",\n\"we'll've\": \"we will have\",\n\"we're\": \"we are\",\n\"we've\": \"we have\",\n\"weren't\": \"were not\",\n\"what'll\": \"what will\",\n\"what'll've\": \"what will have\",\n\"what're\": \"what are\",\n\"what's\": \"what is\",\n\"what've\": \"what have\",\n\"when's\": \"when is\",\n\"when've\": \"when have\",\n\"where'd\": \"where did\",\n\"where's\": \"where is\",\n\"where've\": \"where have\",\n\"who'll\": \"who will\",\n\"who'll've\": \"who will have\",\n\"who's\": \"who is\",\n\"who've\": \"who have\",\n\"why's\": \"why is\",\n\"why've\": \"why have\",\n\"will've\": \"will have\",\n\"won't\": \"will not\",\n\"won't've\": \"will not have\",\n\"would've\": \"would have\",\n\"wouldn't\": \"would not\",\n\"wouldn't've\": \"would not have\",\n\"y'all\": \"you all\",\n\"y'all'd\": \"you all would\",\n\"y'all'd've\": \"you all would have\",\n\"y'all're\": \"you all are\",\n\"y'all've\": \"you all have\",\n\"you'd\": \"you would\",\n\"you'd've\": \"you would have\",\n\"you'll\": \"you will\",\n\"you'll've\": \"you will have\",\n\"you're\": \"you are\",\n\"you've\": \"you have\"\n}\n\nc_re = re.compile('(%s)' % '|'.join(contractions.keys()))\n\ndef expandContractions(text, c_re=c_re):\n    def replace(match):\n        return contractions[match.group(0)]\n    return c_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05eddc246d57b9083e660f5bc871ac4dfc24d8db"},"cell_type":"code","source":"from gensim.parsing.preprocessing import preprocess_string\nfrom gensim.parsing.preprocessing import strip_tags, strip_punctuation, strip_numeric, stem_text\nfrom gensim.parsing.preprocessing import strip_multiple_whitespaces, strip_non_alphanum, remove_stopwords, strip_short\n\nCUSTOM_FILTERS = [lambda x: x.lower(), #lowercase\n                  strip_tags, # remove html tags\n                  strip_punctuation, # replace punctuation with space\n                  strip_multiple_whitespaces,# remove repeating whitespaces\n                  strip_non_alphanum, # remove non-alphanumeric characters\n                  strip_numeric, # remove numbers\n                  remove_stopwords,# remove stopwords\n                  strip_short, # remove words less than minsize=3 characters long\n                  stem_text,\n                 ]\ndef gensim_preprocess(docs):\n    # clean text\n    docs = [expandContractions(doc) for doc in docs]\n    docs = [preprocess_string(text, CUSTOM_FILTERS) for text in docs]\n    # create the bigram and trigram models\n    bigram = models.Phrases(docs, min_count=1, threshold=1)\n    trigram = models.Phrases(bigram[docs], min_count=1, threshold=1)  \n    # phraser is faster\n    bigram_mod = models.phrases.Phraser(bigram)\n    trigram_mod = models.phrases.Phraser(trigram)\n    # apply to docs\n    docs = trigram_mod[bigram_mod[docs]]\n    #docs = [' '.join(text) for text in docs]\n    return docs\n\ntrain_clean = gensim_preprocess(train.question_text)\ntrain_clean[43]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be23d2a586d30420c3919959e02b37b5d908fc10"},"cell_type":"code","source":"# Create Dictionary from our ngram texts containing number of times token appears in training set\ntrain_dictionary = corpora.Dictionary(train_clean)\n\n# filter out extremes\ntrain_dictionary.filter_extremes(no_below=0.1, # filter tokens appearing in <1% of documents\n                                     no_above=0.7, # filter tokens appearing in >70% of documents\n                                     keep_n=100000) # after above filters keep only the 100000 most frequent tokens\n\n# For each document create dictionary with how many words and number of times the words appear\ntrain_corpus = [train_dictionary.doc2bow(text) for text in train_clean]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e01dd3c9b5c6ed526bb8832abaf992e50f3b6845"},"cell_type":"code","source":"# view human readable output\n[[(train_dictionary[id], freq) for id, freq in cp] for cp in train_corpus[:1]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"772ae9dfa7618aca79cb33cf91de326f468ff22b"},"cell_type":"code","source":"# initialize tfidf model\ntfidfi = models.TfidfModel(train_corpus)\n# apply transformation to entire corpus\ntrain_tfidf = tfidfi[train_corpus]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d74fd6010e877a54dfbcecd7d8bbf115852ef930"},"cell_type":"code","source":"# https://radimrehurek.com/gensim/tut2.html#transformation-interface\n# LDA on tfidf\n%time train_lda = models.LdaMulticore(train_tfidf, num_topics=10, id2word=train_dictionary, passes=2, workers=6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bc8840999350b1f9554fb494bfe4a8a550bffac"},"cell_type":"code","source":"train_lda.show_topics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9803c1cce6d9319a5179ba96eb5fd2c46f3e42f1"},"cell_type":"code","source":"# Visualize the topics\npyLDAvis.enable_notebook()\nvis = pyLDAvis.gensim.prepare(train_lda, train_corpus, train_dictionary)\nvis","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4a20663d8c7f7541cd8778379475e89e684dfe9f"},"cell_type":"markdown","source":"From the visualization above we can see that several topics overlap significantly."},{"metadata":{"trusted":true,"_uuid":"ae783ac166551bd49ec52047b54f4da5160996ab"},"cell_type":"code","source":"# using coherence score to find optimal number of topics\n# ref: https://www.machinelearningplus.com/nlp/topic-modeling-gensim-python/\n\ndef compute_coherence_values(dictionary, corpus, texts, limit, start=2, step=3):\n    \"\"\"\n    Compute c_v coherence for various number of topics\n\n    Parameters:\n    ----------\n    dictionary : Gensim dictionary\n    corpus : Gensim corpus\n    texts : List of input texts\n    limit : Max num of topics\n\n    Returns:\n    -------\n    model_list : List of LDA topic models\n    coherence_values : Coherence values corresponding to the LDA model with respective number of topics\n    \"\"\"\n    coherence_values = []\n    model_list = []\n    for num_topics in range(start, limit, step):\n        model = models.LdaMulticore(corpus=corpus, num_topics=num_topics, id2word=dictionary, workers=6, passes=2)\n        model_list.append(model)\n        coherencemodel = models.CoherenceModel(model=model, texts=texts, dictionary=dictionary, coherence='c_v')\n        coherence_values.append(coherencemodel.get_coherence())\n\n    return model_list, coherence_values\n\nmodel_list, coherence_values = compute_coherence_values(dictionary=train_dictionary,\n                                                        corpus=train_tfidf,\n                                                        texts=train_clean,\n                                                        start=2,\n                                                        limit=262,\n                                                        step=20)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b25e217a20af301e44cf73a61e4a36015b60ed9e"},"cell_type":"code","source":"coherence_values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2bb7f0c7cb60f2f09c4f3752772d4ca8a15bf8eb"},"cell_type":"code","source":"# Show graph\nlimit=262; start=2; step=20;\nx = range(start, limit, step)\nsns.lineplot(x, coherence_values)\nsns.despine(left=True, bottom=True)\nplt.title('Training LDA Coherence Scores', fontsize=30)\nplt.xlabel(\"Number of Topics\")\nplt.ylabel(\"Coherence Score\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"88e2b46d95635cdc4fb5740d50f1e65a8fc531ef"},"cell_type":"code","source":"# LDA on tfidf\n%time train_lda = models.LdaMulticore(train_tfidf, num_topics=180, id2word=train_dictionary, passes=2, workers=6)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd1a1e9ad9051ac150b32b2f8f648c73abd5c112"},"cell_type":"code","source":"# Visualize the topics\npyLDAvis.enable_notebook()\nvis = pyLDAvis.gensim.prepare(train_lda, train_tfidf, train_dictionary)\nvis","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4de35735d52e8b109a4c055f6e78163058de39f5"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dcb1cd1ee939a47c260a9de061a1dbe98d9aaa8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}