{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Topic Modeling and Latent Dirichlet Allocation (LDA) in Python"},{"metadata":{},"cell_type":"markdown","source":"Topic modeling is a type of statistical modeling for discovering the abstract “topics” that occur in a collection of documents. Latent Dirichlet Allocation (LDA) is an example of topic model and is used to classify text in a document to a particular topic. It builds a topic per document model and words per topic model, modeled as Dirichlet distributions.\n\nHere we are going to apply LDA to a set of questions and split them into topics. Let’s get started!"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport json\nimport numpy as np \nimport pandas as pd\nimport re\nfrom IPython.display import display\nfrom tqdm import tqdm\nfrom collections import Counter\nimport ast\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nimport plotly as py\nimport plotly.graph_objs as go\nimport gensim\nfrom gensim import corpora, models, similarities\nimport logging\nimport tempfile\nfrom nltk.corpus import stopwords\nfrom string import punctuation\nfrom collections import OrderedDict\nimport seaborn as sns\nimport pyLDAvis.gensim\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\ninit_notebook_mode(connected=True) #do not miss this line\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport matplotlib.pyplot as plt\nimport matplotlib.mlab as mlab\nimport seaborn as sb\n\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom textblob import TextBlob\nimport scipy.stats as stats\n\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.decomposition import LatentDirichletAllocation\nfrom sklearn.manifold import TSNE\n\nfrom bokeh.plotting import figure, output_file, show\nfrom bokeh.models import Label\nfrom bokeh.io import output_notebook\noutput_notebook()\n\n%matplotlib inline\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n        \nfrom sklearn.metrics import accuracy_score, f1_score\nfrom tqdm import tqdm_notebook as tqdm\nfrom Levenshtein import ratio as levenshtein_distance\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction import text\n\nfrom scipy import spatial","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"html_tags = ['<P>', '</P>', '<Table>', '</Table>', '<Tr>', '</Tr>', '<Ul>', '<Ol>', '<Dl>', '</Ul>', '</Ol>', \\\n             '</Dl>', '<Li>', '<Dd>', '<Dt>', '</Li>', '</Dd>', '</Dt>']\nr_buf = ['is', 'are', 'do', 'does', 'did', 'was', 'were', 'will', 'can', 'the', 'a', 'of', 'in', 'and', 'on', \\\n         'what', 'where', 'when', 'which'] + html_tags\n\ndef clean(x):\n    x = x.lower()\n    for r in r_buf:\n        x = x.replace(r, '')\n    x = re.sub(' +', ' ', x)\n    return x\n\nbin_question_tokens = ['is', 'are', 'do', 'does', 'did', 'was', 'were', 'will', 'can']\nstop_words = text.ENGLISH_STOP_WORDS.union([\"book\"])\n\ndef predict(json_data, annotated=False):\n    # Parse JSON data\n    candidates = json_data['long_answer_candidates']\n    candidates = [c for c in candidates if c['top_level'] == True]\n    doc_tokenized = json_data['document_text'].split(' ')\n    question = json_data['question_text']\n    question_s = question.split(' ') \n    if annotated:\n        ann = json_data['annotations'][0]\n\n    # TFIDF for the document\n    tfidf = TfidfVectorizer(ngram_range=(1,1), stop_words=stop_words)\n    tfidf.fit([json_data['document_text']])\n    q_tfidf = tfidf.transform([question]).todense()\n\n    # Find the nearest answer from candidates\n    distances = []\n    scores = []\n    i_ann = -1\n    for i, c in enumerate(candidates):\n        s, e = c['start_token'], c['end_token']\n        t = ' '.join(doc_tokenized[s:e])\n        distances.append(levenshtein_distance(clean(question), clean(t)))\n        \n        t_tfidf = tfidf.transform([t]).todense()\n        score = 1 - spatial.distance.cosine(q_tfidf, t_tfidf)\n        \n#         score = 0\n        \n#         for w in doc_tokenized[s:e]:\n#             if w in q_s:\n#                 score += 0.1\n\n        scores.append(score)\n\n    # Format results\n#     ans = candidates[np.argmin(distances)]\n    ans = candidates[np.argmax(scores)]\n    if np.max(scores) < 0.2:\n        ans_long = '-1:-1'\n    else:\n        ans_long = str(ans['start_token']) + ':' + str(ans['end_token'])\n    if question_s[0] in bin_question_tokens:\n        ans_short = 'YES'\n    else:\n        ans_short = ''\n        \n    # Preparing data for debug\n    if annotated:\n        ann_long_text = ' '.join(doc_tokenized[ann['long_answer']['start_token']:ann['long_answer']['end_token']])\n        if ann['yes_no_answer'] == 'NONE':\n            if len(json_data['annotations'][0]['short_answers']) > 0:\n                ann_short_text = ' '.join(doc_tokenized[ann['short_answers'][0]['start_token']:ann['short_answers'][0]['end_token']])\n            else:\n                ann_short_text = ''\n        else:\n            ann_short_text = ann['yes_no_answer']\n    else:\n        ann_long_text = ''\n        ann_short_text = ''\n        \n    ans_long_text = ' '.join(doc_tokenized[ans['start_token']:ans['end_token']])\n    if len(ans_short) > 0 or ans_short == 'YES':\n        ans_short_text = ans_short\n    else:\n        ans_short_text = '' # Fix when short answers will work\n                    \n    return ans_long, ans_short, question, ann_long_text, ann_short_text, ans_long_text, ans_short_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nids = []\nanns = []\npreds = []\n\n# Debug data\nquestions = []\nann_texts = []\nans_texts = []\n\nn_samples = 500\n\nwith open('/kaggle/input/tensorflow2-question-answering/simplified-nq-train.jsonl', 'r') as json_file:\n    cnt = 0\n    for line in tqdm(json_file):\n        json_data = json.loads(line)\n        \n        ids.append(str(json_data['example_id']) + '_long')\n        ids.append(str(json_data['example_id']) + '_short')\n        \n        l_ans = str(json_data['annotations'][0]['long_answer']['start_token']) + ':' + \\\n            str(json_data['annotations'][0]['long_answer']['end_token'])\n        if json_data['annotations'][0]['yes_no_answer'] == 'NONE':\n            if len(json_data['annotations'][0]['short_answers']) > 0:\n                s_ans = str(json_data['annotations'][0]['short_answers'][0]['start_token']) + ':' + \\\n                    str(json_data['annotations'][0]['short_answers'][0]['end_token'])\n            else:\n                s_ans = ''\n        else:\n            s_ans = json_data['annotations'][0]['yes_no_answer']\n            \n        anns.append(l_ans)\n        anns.append(s_ans)\n        \n        l_ans, s_ans, question, ann_long_text, ann_short_text, ans_long_text, ans_short_text = predict(json_data, annotated=True)\n        \n        preds.append(l_ans)\n        preds.append(s_ans)\n        questions.append(question)\n        questions.append(question)\n        ann_texts.append(ann_long_text)\n        ann_texts.append(ann_short_text)\n        ans_texts.append(ans_long_text)\n        ans_texts.append(ans_short_text)\n        \n        cnt += 1\n        if cnt >= n_samples:\n            break\n        \ntrain_ann = pd.DataFrame()\ntrain_ann['example_id'] = ids\ntrain_ann['question'] = questions\ntrain_ann['CorrectString'] = anns\ntrain_ann['CorrectText'] = ann_texts\nif len(preds) > 0:\n    train_ann['PredictionString'] = preds\n    train_ann['PredictionText'] = ans_texts\n    \ntrain_ann.to_csv('train_data.csv', index=False)\ntrain_ann.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ann.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**First we develop a list of the top words used across all questions, giving us a glimpse into the core vocabulary of the source data. Stop words are omitted here to avoid any trivial conjunctions, prepositions, etc.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"reindexed_data = train_ann['question']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define helper functions\ndef get_top_n_words(n_top_words, count_vectorizer, text_data):\n    '''\n    returns a tuple of the top n words in a sample and their \n    accompanying counts, given a CountVectorizer object and text sample\n    '''\n    vectorized_headlines = count_vectorizer.fit_transform(text_data.values)\n    vectorized_total = np.sum(vectorized_headlines, axis=0)\n    word_indices = np.flip(np.argsort(vectorized_total)[0,:], 1)\n    word_values = np.flip(np.sort(vectorized_total)[0,:],1)\n    \n    word_vectors = np.zeros((n_top_words, vectorized_headlines.shape[1]))\n    for i in range(n_top_words):\n        word_vectors[i,word_indices[0,i]] = 1\n\n    words = [word[0].encode('ascii').decode('utf-8') for \n             word in count_vectorizer.inverse_transform(word_vectors)]\n\n    return (words, word_values[0,:n_top_words].tolist()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_vectorizer = CountVectorizer(stop_words='english')\nwords, word_values = get_top_n_words(n_top_words=15,\n                                     count_vectorizer=count_vectorizer, \n                                     text_data=reindexed_data)\n\nfig, ax = plt.subplots(figsize=(16,8))\nax.bar(range(len(words)), word_values);\nax.set_xticks(range(len(words)));\nax.set_xticklabels(words, rotation='vertical');\nax.set_title('Top words in headlines dataset (excluding stop words)');\nax.set_xlabel('Word');\nax.set_ylabel('Number of occurences');\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Next we generate a histogram of headline word lengths, and use part-of-speech tagging to understand the types of words used across the corpus. This requires first converting all headline strings to TextBlobs and calling the pos_tags method on each, yielding a list of tagged words for each headline. A complete list of such word tags is available [here](https://www.clips.uantwerpen.be/pages/MBSP-tags)"},{"metadata":{"trusted":true},"cell_type":"code","source":"tagged_headlines = [TextBlob(reindexed_data[i]).pos_tags for i in range(reindexed_data.shape[0])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tagged_headlines_df = pd.DataFrame({'tags':tagged_headlines})\n\nword_counts = [] \npos_counts = {}\n\nfor headline in tagged_headlines_df[u'tags']:\n    word_counts.append(len(headline))\n    for tag in headline:\n        if tag[1] in pos_counts:\n            pos_counts[tag[1]] += 1\n        else:\n            pos_counts[tag[1]] = 1\n            \nprint('Total number of words: ', np.sum(word_counts))\nprint('Mean number of words per question: ', np.mean(word_counts))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = stats.norm.pdf(np.linspace(0,14,50), np.mean(word_counts), np.std(word_counts))\n\nfig, ax = plt.subplots(figsize=(18,8))\nax.hist(word_counts, bins=range(1,14), density=True);\nax.plot(np.linspace(0,14,50), y, 'r--', linewidth=1);\nax.set_title('Headline word lengths');\nax.set_xticks(range(1,14));\nax.set_xlabel('Number of words');\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_sorted_types = sorted(pos_counts, key=pos_counts.__getitem__, reverse=True)\npos_sorted_counts = sorted(pos_counts.values(), reverse=True)\n\nfig, ax = plt.subplots(figsize=(18,8))\nax.bar(range(len(pos_counts)), pos_sorted_counts);\nax.set_xticks(range(len(pos_counts)));\nax.set_xticklabels(pos_sorted_types);\nax.set_title('Part-of-Speech Tagging for Keywords Corpus');\nax.set_xlabel('Type of Word');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Topic Modelling\n\nWe now apply a clustering algorithm to the headlines corpus in order to study the topic focus of ABC News, as well as how it has evolved through time. To do so, we first experiment with a small subsample of the dataset in order to determine which of the two potential clustering algorithms is most appropriate – once this has been ascertained, we then scale up to a larger portion of the available data."},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing\nThe only preprocessing step required in our case is feature construction, where we take the sample of text headlines and represent them in some tractable feature space. In practice, this simply means converting each string to a numerical vector. This can be done using the CountVectorizer object from SKLearn, which yields an $n×K$ document-term matrix where $K$ is the number of distinct words across the $n$ headlines in our sample (less stop words and with a limit of max_features)."},{"metadata":{"trusted":true},"cell_type":"code","source":"small_count_vectorizer = CountVectorizer(stop_words='english', max_features=40000)\nsmall_text_sample = reindexed_data.sample(n=500, random_state=0).values\n\nprint('Questions before vectorization: {}'.format(small_text_sample[123]))\n\nsmall_document_term_matrix = small_count_vectorizer.fit_transform(small_text_sample)\n\nprint('Questions after vectorization: \\n{}'.format(small_document_term_matrix[123]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Thus we have our (very high-rank and sparse) training data, small_document_term_matrix, and can now actually implement a clustering algorithm. Our choice will be either Latent Semantic Analysis or Latent Dirichilet Allocation. Both will take our document-term matrix as input and yield an $n \\times N$ topic matrix as output, where $N$ is the number of topic categories (which we supply as a parameter). For the moment, we shall take this to be 8."},{"metadata":{"trusted":true},"cell_type":"code","source":"n_topics = 15","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Latent Semantic Analysis\n\nLet's start by experimenting with LSA. This is effectively just a truncated singular value decomposition of a (very high-rank and sparse) document-term matrix, with only the $r=$n_topics largest singular values preserved."},{"metadata":{"trusted":true},"cell_type":"code","source":"lsa_model = TruncatedSVD(n_components=n_topics)\nlsa_topic_matrix = lsa_model.fit_transform(small_document_term_matrix)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Taking the $\\arg \\max$ of each headline in this topic matrix will give the predicted topics of each headline in the sample. We can then sort these into counts of each topic."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define helper functions\ndef get_keys(topic_matrix):\n    '''\n    returns an integer list of predicted topic \n    categories for a given topic matrix\n    '''\n    keys = topic_matrix.argmax(axis=1).tolist()\n    return keys\n\ndef keys_to_counts(keys):\n    '''\n    returns a tuple of topic categories and their \n    accompanying magnitudes for a given list of keys\n    '''\n    count_pairs = Counter(keys).items()\n    categories = [pair[0] for pair in count_pairs]\n    counts = [pair[1] for pair in count_pairs]\n    return (categories, counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lsa_keys = get_keys(lsa_topic_matrix)\nlsa_categories, lsa_counts = keys_to_counts(lsa_keys)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**However, these topic categories are in and of themselves a little meaningless. In order to better characterise them, it will be helpful to find the most frequent words in each.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define helper functions\ndef get_top_n_words(n, keys, document_term_matrix, count_vectorizer):\n    '''\n    returns a list of n_topic strings, where each string contains the n most common \n    words in a predicted category, in order\n    '''\n    top_word_indices = []\n    for topic in range(n_topics):\n        temp_vector_sum = 0\n        for i in range(len(keys)):\n            if keys[i] == topic:\n                temp_vector_sum += document_term_matrix[i]\n        temp_vector_sum = temp_vector_sum.toarray()\n        top_n_word_indices = np.flip(np.argsort(temp_vector_sum)[0][-n:],0)\n        top_word_indices.append(top_n_word_indices)   \n    top_words = []\n    for topic in top_word_indices:\n        topic_words = []\n        for index in topic:\n            temp_word_vector = np.zeros((1,document_term_matrix.shape[1]))\n            temp_word_vector[:,index] = 1\n            the_word = count_vectorizer.inverse_transform(temp_word_vector)[0][0]\n            topic_words.append(the_word.encode('ascii').decode('utf-8'))\n        top_words.append(\" \".join(topic_words))         \n    return top_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_n_words_lsa = get_top_n_words(10, lsa_keys, small_document_term_matrix, small_count_vectorizer)\n\nfor i in range(len(top_n_words_lsa)):\n    print(\"Topic {}: \".format(i+1), top_n_words_lsa[i])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Thus we have converted our initial small sample of headlines into a list of predicted topic categories, where each category is characterised by its most frequent words. The relative magnitudes of each of these categories can then be easily visualised though use of a bar chart."},{"metadata":{"trusted":true},"cell_type":"code","source":"top_3_words = get_top_n_words(3, lsa_keys, small_document_term_matrix, small_count_vectorizer)\nlabels = ['Topic {}: \\n'.format(i) + top_3_words[i] for i in lsa_categories]\n\nfig, ax = plt.subplots(figsize=(16,8))\nax.bar(lsa_categories, lsa_counts);\nax.set_xticks(lsa_categories);\nax.set_xticklabels(labels);\nax.set_ylabel('Number of questions');\nax.set_title('LSA topic counts');\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"However, this does not provide a great point of comparison with other clustering algorithms. In order to properly contrast LSA with LDA we instead use a dimensionality-reduction technique called $t$-SNE, which will also serve to better illuminate the success of the clustering process."},{"metadata":{"trusted":true},"cell_type":"code","source":"tsne_lsa_model = TSNE(n_components=2, perplexity=50, learning_rate=100, \n                        n_iter=2000, verbose=1, random_state=0, angle=0.75)\ntsne_lsa_vectors = tsne_lsa_model.fit_transform(lsa_topic_matrix)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now that we have reduced these n_topics-dimensional vectors to two-dimensional representations, we can then plot the clusters using Bokeh. Before doing so however, it will be useful to derive the centroid location of each topic, so as to better contextualise our visualisation."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define helper functions\ndef get_mean_topic_vectors(keys, two_dim_vectors):\n    '''\n    returns a list of centroid vectors from each predicted topic category\n    '''\n    mean_topic_vectors = []\n    for t in range(n_topics):\n        articles_in_that_topic = []\n        for i in range(len(keys)):\n            if keys[i] == t:\n                articles_in_that_topic.append(two_dim_vectors[i])    \n        \n        articles_in_that_topic = np.vstack(articles_in_that_topic)\n        mean_article_in_that_topic = np.mean(articles_in_that_topic, axis=0)\n        mean_topic_vectors.append(mean_article_in_that_topic)\n    return mean_topic_vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"colormap = np.array([\n    \"#1f77b4\", \"#aec7e8\", \"#ff7f0e\", \"#ffbb78\", \"#2ca02c\",\n    \"#98df8a\", \"#d62728\", \"#ff9896\", \"#9467bd\", \"#c5b0d5\",\n    \"#8c564b\", \"#c49c94\", \"#e377c2\", \"#f7b6d2\", \"#7f7f7f\",\n    \"#c7c7c7\", \"#bcbd22\", \"#dbdb8d\", \"#17becf\", \"#9edae5\" ])\ncolormap = colormap[:n_topics]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**All that remains is to plot the clustered questions. Also included are the top three words in each cluster, which are placed at the centroid for that topic.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"top_3_words_lsa = get_top_n_words(3, lsa_keys, small_document_term_matrix, small_count_vectorizer)\nlsa_mean_topic_vectors = get_mean_topic_vectors(lsa_keys, tsne_lsa_vectors)\n\nplot = figure(title=\"t-SNE Clustering of {} LSA Topics\".format(n_topics), plot_width=700, plot_height=700)\nplot.scatter(x=tsne_lsa_vectors[:,0], y=tsne_lsa_vectors[:,1], color=colormap[lsa_keys])\n\nfor t in range(n_topics):\n    label = Label(x=lsa_mean_topic_vectors[t][0], y=lsa_mean_topic_vectors[t][1], \n                  text=top_3_words_lsa[t], text_color=colormap[t])\n    plot.add_layout(label)\n    \nshow(plot)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Evidently, this is a bit a of a failed result. We have failed to reach any great degree of separation across the topic categories, and it is difficult to tell whether this can be attributed to the LSA decomposition or instead the $t$-SNE dimensionality reduction process. Let's move forward and try another clustering technique."},{"metadata":{},"cell_type":"markdown","source":"# Latent Dirichilet Allocation\n\nWe now repeat this process using LDA instead of LSA. LDA is instead a generative probabilistic process, designed with the specific goal of uncovering latent topic structure in text corpora."},{"metadata":{"trusted":true},"cell_type":"code","source":"lda_model = LatentDirichletAllocation(n_components=n_topics, learning_method='online', \n                                          random_state=0, verbose=0)\nlda_topic_matrix = lda_model.fit_transform(small_document_term_matrix)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Once again, we take the $\\arg \\max$ of each entry in the topic matrix to obtain the predicted topic category for each question. These topic categories can then be characterised by their most frequent words."},{"metadata":{"trusted":true},"cell_type":"code","source":"lda_keys = get_keys(lda_topic_matrix)\nlda_categories, lda_counts = keys_to_counts(lda_keys)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_n_words_lda = get_top_n_words(10, lda_keys, small_document_term_matrix, small_count_vectorizer)\n\nfor i in range(len(top_n_words_lda)):\n    print(\"Topic {}: \".format(i+1), top_n_words_lda[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_3_words = get_top_n_words(3, lda_keys, small_document_term_matrix, small_count_vectorizer)\nlabels = ['Topic {}: \\n'.format(i) + top_3_words[i] for i in lda_categories]\n\nfig, ax = plt.subplots(figsize=(16,8))\nax.bar(lda_categories, lda_counts);\nax.set_xticks(lda_categories);\nax.set_xticklabels(labels);\nax.set_title('LDA topic counts');\nax.set_ylabel('Number of questions');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**However, in order to properly compare LDA with LSA, we again take this topic matrix and project it into two dimensions with $t$-SNE.**"},{"metadata":{"trusted":true},"cell_type":"code","source":"tsne_lda_model = TSNE(n_components=2, perplexity=50, learning_rate=100, \n                        n_iter=2000, verbose=1, random_state=0, angle=0.75)\ntsne_lda_vectors = tsne_lda_model.fit_transform(lda_topic_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_3_words_lda = get_top_n_words(3, lda_keys, small_document_term_matrix, small_count_vectorizer)\nlda_mean_topic_vectors = get_mean_topic_vectors(lda_keys, tsne_lda_vectors)\n\nplot = figure(title=\"t-SNE Clustering of {} LDA Topics\".format(n_topics), plot_width=700, plot_height=700)\nplot.scatter(x=tsne_lda_vectors[:,0], y=tsne_lda_vectors[:,1], color=colormap[lda_keys])\n\nfor t in range(n_topics):\n    label = Label(x=lda_mean_topic_vectors[t][0], y=lda_mean_topic_vectors[t][1], \n                  text=top_3_words_lda[t], text_color=colormap[t])\n    plot.add_layout(label)\n\nshow(plot)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is a much better result! Controlling for $t$-SNE, it would seem that LDA has had much more succcess than LSA in separating out the topic categories. For this reason, LDA appears the more appropriate algorithm when we scale up the clustering process."},{"metadata":{},"cell_type":"markdown","source":"# LDA with an other way of visualisation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preparing a corpus for analysis and checking the first 10 entries\ncorpus=[]\n\ncorpus = train_ann['question'].to_list()\n\ncorpus[:10]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**There is many repetitive questions, we will delate these duplicates using set function**"},{"metadata":{"trusted":true},"cell_type":"code","source":"corpus = list(set(corpus))\ncorpus[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('There is '+str(len(corpus))+' unique question')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEMP_FOLDER = tempfile.gettempdir()\nprint('Folder \"{}\" will be used to save temporary dictionary and corpus.'.format(TEMP_FOLDER))\n\nlogging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# removing common words and tokenizing\nstoplist = stopwords.words('english') + list(punctuation) + list(\"([)]?\") + [\")?\"]\n\ntexts = [[word for word in str(document).lower().split() if word not in stoplist] for document in corpus]\n\ndictionary = corpora.Dictionary(texts)\ndictionary.save(os.path.join(TEMP_FOLDER, 'TF2.0_QA.dict'))  # store the dictionary,","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corpus = [dictionary.doc2bow(text) for text in texts]\ncorpora.MmCorpus.serialize(os.path.join(TEMP_FOLDER, 'TF2.0_QA.mm'), corpus) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### In the previous cells, I created a corpus of documents represented as a stream of vectors. To continue, lets use that corpus, with the help of Gensim."},{"metadata":{},"cell_type":"markdown","source":"# Creating a transformation\n\nThe transformations are standard Python objects, typically initialized by means of a training corpus:\n\nDifferent transformations may require different initialization parameters; in case of TfIdf, the “training” consists simply of going through the supplied corpus once and computing document frequencies of all its features. Training other models, such as Latent Semantic Analysis or Latent Dirichlet Allocation, is much more involve"},{"metadata":{"trusted":true},"cell_type":"code","source":"tfidf = models.TfidfModel(corpus) # step 1 -- initialize a model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Note\n\nTransformations always convert between two specific vector spaces. The same vector space (= the same set of feature ids) must be used for training as well as for subsequent vector transformations. Failure to use the same input feature space, such as applying a different string preprocessing, using different feature ids, or using bag-of-words input vectors where TfIdf vectors are expected, will result in feature mismatch during transformation calls and consequently in either garbage output and/or runtime exceptions."},{"metadata":{},"cell_type":"markdown","source":"**From now on, tfidf is treated as a read-only object that can be used to apply a transformation to a whole corpus:**\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"corpus_tfidf = tfidf[corpus]  # step 2 -- use the model to transform vectors","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LDA:\n\nLatent Dirichlet Allocation, LDA is yet another transformation from bag-of-words counts into a topic space of lower dimensionality. LDA is a probabilistic extension of LSA (also called multinomial PCA), so LDA’s topics can be interpreted as probability distributions over words. These distributions are, just like with LSA, inferred automatically from a training corpus. Documents are in turn interpreted as a (soft) mixture of these topics (again, just like with LSA)."},{"metadata":{},"cell_type":"markdown","source":"**I will try 15 topics**"},{"metadata":{"trusted":true},"cell_type":"code","source":"total_topics = 15","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lda = models.LdaModel(corpus, id2word=dictionary, num_topics=total_topics)\ncorpus_lda = lda[corpus_tfidf] # create a double wrapper over the original corpus: bow->tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Show first n important word in the topics:\n"},{"metadata":{"trusted":true},"cell_type":"code","source":"lda.show_topics(total_topics,5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_lda = {i: OrderedDict(lda.show_topic(i,25)) for i in range(total_topics)}\n#data_lda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_lda = pd.DataFrame(data_lda)\ndf_lda = df_lda.fillna(0).T\nprint(df_lda.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_lda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g=sns.clustermap(df_lda.corr(), center=0, standard_scale=1, cmap=\"OrRd\", metric='cosine', linewidths=.75, figsize=(15, 15))\nplt.setp(g.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)\nplt.show()\n#plt.setp(ax_heatmap.get_yticklabels(), rotation=0)  # For y axis","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pyLDAvis.enable_notebook()\npanel = pyLDAvis.gensim.prepare(lda, corpus_lda, dictionary, mds='tsne')\npanel","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}