{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\npd.set_option('display.max_columns', 50)\npd.set_option('display.max_rows', 150)\nimport os\nimport gc\ngc.enable()\nimport time\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\nfrom scipy.sparse import hstack\nfrom scipy import stats\n%matplotlib inline\nfrom datetime import timedelta\nimport datetime as dt\nimport matplotlib.pyplot as plt\nplt.rcParams['figure.figsize'] = [16, 10]\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.decomposition import PCA\nfrom sklearn.cluster import MiniBatchKMeans\nimport warnings\nwarnings.filterwarnings('ignore')\nimport urllib        #for url stuff\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction import text\nfrom IPython.display import display\nfrom tqdm import tqdm\nfrom collections import Counter\nimport ast\n\n\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom textblob import TextBlob\nimport scipy.stats as stats\n\nimport seaborn as sns\nfrom sklearn.manifold import TSNE\nfrom sklearn.decomposition import PCA, TruncatedSVD\nimport matplotlib.patches as mpatches\nimport time\n\nimport seaborn as sns #for making plots\nimport matplotlib.pyplot as plt # for plotting\nimport os  # for os commands\n\nimport gensim\nfrom gensim import corpora, models, similarities\nimport logging\nimport tempfile\nfrom nltk.corpus import stopwords\nfrom string import punctuation\nfrom collections import OrderedDict\n\nfrom sklearn.decomposition import TruncatedSVD\nfrom sklearn.decomposition import LatentDirichletAllocation\nfrom sklearn.manifold import TSNE\n\n\nfrom bokeh.plotting import figure, output_file, show\nfrom bokeh.models import Label\nfrom bokeh.io import output_notebook\noutput_notebook()\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"## Common Variables for Notebook \nROOT = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'\n\n## load the data \ndf_train = pd.read_csv(ROOT+'jigsaw-toxic-comment-train.csv')\ndf_test = pd.read_csv(ROOT+'test.csv')\ndf_sub = pd.read_csv(ROOT+'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# First look at the data"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Looking data format and types\nprint(df_train.info())\nprint(df_test.info())\nprint(df_sub.info())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Some Statistics\ndf_train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Take a look at the data\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploring the data:\n## Let's check target distrubution\n## Check for Class Imbalance"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.columns","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Toxic\t"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"target = df_train['toxic']\nsns.set_style('whitegrid')\nsns.countplot(target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Severe toxic"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = df_train['severe_toxic']\nsns.set_style('whitegrid')\nsns.countplot(target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Obscene"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = df_train['obscene']\nsns.set_style('whitegrid')\nsns.countplot(target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Threat"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = df_train['threat']\nsns.set_style('whitegrid')\nsns.countplot(target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Insult"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = df_train['insult']\nsns.set_style('whitegrid')\nsns.countplot(target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Identity hate"},{"metadata":{"trusted":true},"cell_type":"code","source":"target = df_train['identity_hate']\nsns.set_style('whitegrid')\nsns.countplot(target)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Check Text Content"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train[\"comment_text\"].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#To check the text comment we can use a list\ndf_train[\"comment_text\"].tolist()[:5]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's make it more perfect"},{"metadata":{"trusted":true},"cell_type":"code","source":"t = df_train[\"comment_text\"].to_list()\nfor i in range(5):\n    print('-'*105)\n    print('Comment '+str(i+1)+': '+t[i])\n    print('-'*105)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Let's do some text analysis"},{"metadata":{},"cell_type":"markdown","source":"## I want to check if there is some duplicated comments."},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train.shape[0])\nprint(len(set(df_train['comment_text'])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### There is not any duplicated comment for training set, what about test set?"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_test.shape[0])\nprint(len(set(df_test['content'])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Topic Modeling"},{"metadata":{},"cell_type":"markdown","source":"## I thinks it's very important to do topic modeling and understand the content of each comment.\n## We will start with some Topic Modeling Algorithms like Latent Dirichlet Allocation (LDA)\n## Topic modeling is a type of statistical modeling for discovering the abstract “topics” that occur in a collection of documents. Latent Dirichlet Allocation (LDA) is an example of topic model and is used to classify text in a document to a particular topic. It builds a topic per document model and words per topic model, modeled as Dirichlet distributions. Here we are going to apply LDA to the cmments and split them into topics."},{"metadata":{"trusted":true},"cell_type":"code","source":"html_tags = ['<P>', '</P>', '<Table>', '</Table>', '<Tr>', '</Tr>', '<Ul>', '<Ol>', '<Dl>', '</Ul>', '</Ol>', \\\n             '</Dl>', '<Li>', '<Dd>', '<Dt>', '</Li>', '</Dd>', '</Dt>']\nr_buf = ['It', 'is', 'are', 'do', 'does', 'did', 'was', 'were', 'will', 'can', 'the', 'a', 'of', 'in', 'and', 'on', \\\n         'what', 'where', 'when', 'which'] + html_tags\n\ndef clean(x):\n    x = x.lower()\n    for r in r_buf:\n        x = x.replace(r, '')\n    x = re.sub(' +', ' ', x)\n    return x\n\nbin_question_tokens = ['it', 'is', 'are', 'do', 'does', 'did', 'was', 'were', 'will', 'can']\nstop_words = text.ENGLISH_STOP_WORDS.union([\"book\"])\n\ndef predict(json_data, annotated=False):\n    # Parse JSON data\n    candidates = json_data['long_answer_candidates']\n    candidates = [c for c in candidates if c['top_level'] == True]\n    doc_tokenized = json_data['document_text'].split(' ')\n    question = json_data['question_text']\n    question_s = question.split(' ') \n    if annotated:\n        ann = json_data['annotations'][0]\n\n    # TFIDF for the document\n    tfidf = TfidfVectorizer(ngram_range=(1,1), stop_words=stop_words)\n    tfidf.fit([json_data['document_text']])\n    q_tfidf = tfidf.transform([question]).todense()\n\n    # Find the nearest answer from candidates\n    distances = []\n    scores = []\n    i_ann = -1\n    for i, c in enumerate(candidates):\n        s, e = c['start_token'], c['end_token']\n        t = ' '.join(doc_tokenized[s:e])\n        distances.append(levenshtein_distance(clean(question), clean(t)))\n        \n        t_tfidf = tfidf.transform([t]).todense()\n        score = 1 - spatial.distance.cosine(q_tfidf, t_tfidf)\n        \n#         score = 0\n        \n#         for w in doc_tokenized[s:e]:\n#             if w in q_s:\n#                 score += 0.1\n\n        scores.append(score)\n\n    # Format results\n#     ans = candidates[np.argmin(distances)]\n    ans = candidates[np.argmax(scores)]\n    if np.max(scores) < 0.2:\n        ans_long = '-1:-1'\n    else:\n        ans_long = str(ans['start_token']) + ':' + str(ans['end_token'])\n    if question_s[0] in bin_question_tokens:\n        ans_short = 'YES'\n    else:\n        ans_short = ''\n        \n    # Preparing data for debug\n    if annotated:\n        ann_long_text = ' '.join(doc_tokenized[ann['long_answer']['start_token']:ann['long_answer']['end_token']])\n        if ann['yes_no_answer'] == 'NONE':\n            if len(json_data['annotations'][0]['short_answers']) > 0:\n                ann_short_text = ' '.join(doc_tokenized[ann['short_answers'][0]['start_token']:ann['short_answers'][0]['end_token']])\n            else:\n                ann_short_text = ''\n        else:\n            ann_short_text = ann['yes_no_answer']\n    else:\n        ann_long_text = ''\n        ann_short_text = ''\n        \n    ans_long_text = ' '.join(doc_tokenized[ans['start_token']:ans['end_token']])\n    if len(ans_short) > 0 or ans_short == 'YES':\n        ans_short_text = ans_short\n    else:\n        ans_short_text = '' # Fix when short answers will work\n                    \n    return ans_long, ans_short, question, ann_long_text, ann_short_text, ans_long_text, ans_short","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reindexed_data = df_train['comment_text'] #.str.decode(\"utf-8\", errors=\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define helper functions\ndef get_top_n_words(n_top_words, count_vectorizer, text_data):\n    '''\n    returns a tuple of the top n words in a sample and their \n    accompanying counts, given a CountVectorizer object and text sample\n    '''\n    vectorized_headlines = count_vectorizer.fit_transform(text_data.values)\n    vectorized_total = np.sum(vectorized_headlines, axis=0)\n    word_indices = np.flip(np.argsort(vectorized_total)[0,:], 1)\n    word_values = np.flip(np.sort(vectorized_total)[0,:],1)\n    \n    word_vectors = np.zeros((n_top_words, vectorized_headlines.shape[1]))\n    for i in range(n_top_words):\n        word_vectors[i,word_indices[0,i]] = 1\n\n    words = [word[0].encode('ascii',errors=\"ignore\").decode('utf-8',errors=\"ignore\") for \n             word in count_vectorizer.inverse_transform(word_vectors)]\n\n    return (words, word_values[0,:n_top_words].tolist()[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_vectorizer = CountVectorizer(stop_words='english')\nwords, word_values = get_top_n_words(n_top_words=25,\n                                     count_vectorizer=count_vectorizer, \n                                     text_data=reindexed_data)\n\nfig, ax = plt.subplots(figsize=(10,4))\nax.bar(range(len(words)), word_values);\nax.set_xticks(range(len(words)));\nax.set_xticklabels(words, rotation='vertical');\nax.set_title('Top words in headlines dataset (excluding stop words)');\nax.set_xlabel('Word');\nax.set_ylabel('Number of occurences');\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Next we generate a histogram of headline word lengths, and use part-of-speech tagging to understand the types of words used across the corpus. This requires first converting all headline strings to TextBlobs and calling the pos_tags method on each, yielding a list of tagged words for each headline."},{"metadata":{"trusted":true},"cell_type":"code","source":"tagged_headlines = [TextBlob(reindexed_data[i]).pos_tags for i in range(reindexed_data.shape[0])]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tagged_headlines_df = pd.DataFrame({'tags':tagged_headlines})\n\nword_counts = [] \npos_counts = {}\n\nfor headline in tagged_headlines_df[u'tags']:\n    word_counts.append(len(headline))\n    for tag in headline:\n        if tag[1] in pos_counts:\n            pos_counts[tag[1]] += 1\n        else:\n            pos_counts[tag[1]] = 1\n            \nprint('Total number of words: ', np.sum(word_counts))\nprint('Mean number of words per comment: ', np.mean(word_counts))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = stats.norm.pdf(np.linspace(0,14,50), np.mean(word_counts), np.std(word_counts))\n\nfig, ax = plt.subplots(figsize=(8,4))\nax.hist(word_counts, bins=range(1,14), density=True);\nax.plot(np.linspace(0,14,50), y, 'r--', linewidth=1);\nax.set_title('Headline word lengths');\nax.set_xticks(range(1,14));\nax.set_xlabel('Number of words');\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part-of-Speech Tagging for comments Corpus"},{"metadata":{"trusted":true},"cell_type":"code","source":"pos_sorted_types = sorted(pos_counts, key=pos_counts.__getitem__, reverse=True)\npos_sorted_counts = sorted(pos_counts.values(), reverse=True)\n\nfig, ax = plt.subplots(figsize=(14,4))\nax.bar(range(len(pos_counts)), pos_sorted_counts);\nax.set_xticks(range(len(pos_counts)));\nax.set_xticklabels(pos_sorted_types);\nax.set_title('Part-of-Speech Tagging for comments Corpus');\nax.set_xlabel('Type of Word');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Topic Modelling\n\n#### Now we will apply a clustering algorithm to the headlines corpus in order to study the topic, as well as how it has evolved through time. To do so, we first experiment with a small subsample of the dataset in order to determine which of the two potential clustering algorithms is most appropriate – once this has been ascertained, we then scale up to a larger portion of the available data.\n\n## Preprocessing\n\n#### The only preprocessing step required in our case is feature construction, where we take the sample of text comments and represent them in some tractable feature space. In practice, this simply means converting each string to a numerical vector. This can be done using the CountVectorizer object from SKLearn, which yields an  n×K  document-term matrix where  K  is the number of distinct words across the  n  headlines in our sample (less stop words and with a limit of max_features)."},{"metadata":{"trusted":true},"cell_type":"code","source":"small_count_vectorizer = CountVectorizer(stop_words='english', max_features=40000)\nsmall_text_sample = reindexed_data.sample(n=500, random_state=0).values\n\nprint('Cooments before vectorization: {}'.format(small_text_sample[123]))\n\nsmall_document_term_matrix = small_count_vectorizer.fit_transform(small_text_sample)\n\nprint('Cooments after vectorization: \\n{}'.format(small_document_term_matrix[123]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Thus we have our (very high-rank and sparse) training data, small_document_term_matrix, and can now actually implement a clustering algorithm. Our choice will be either Latent Semantic Analysis or Latent Dirichilet Allocation. Both will take our document-term matrix as input and yield an  n×N  topic matrix as output, where  N  is the number of topic categories (which we supply as a parameter). For the moment, we shall take this to be 5 like categories number."},{"metadata":{"trusted":true},"cell_type":"code","source":"#number of topics\nn_topics = 5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Latent Semantic Analysis (LSA)\n## Let's start by experimenting with LSA. This is effectively just a truncated singular value decomposition of a (very high-rank and sparse) document-term matrix, with only the  r= n_topics largest singular values preserved."},{"metadata":{"trusted":true},"cell_type":"code","source":"lsa_model = TruncatedSVD(n_components=n_topics)\nlsa_topic_matrix = lsa_model.fit_transform(small_document_term_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define helper functions\ndef get_keys(topic_matrix):\n    '''\n    returns an integer list of predicted topic \n    categories for a given topic matrix\n    '''\n    keys = topic_matrix.argmax(axis=1).tolist()\n    return keys\n\ndef keys_to_counts(keys):\n    '''\n    returns a tuple of topic categories and their \n    accompanying magnitudes for a given list of keys\n    '''\n    count_pairs = Counter(keys).items()\n    categories = [pair[0] for pair in count_pairs]\n    counts = [pair[1] for pair in count_pairs]\n    return (categories, counts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lsa_keys = get_keys(lsa_topic_matrix)\nlsa_categories, lsa_counts = keys_to_counts(lsa_keys)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define helper functions\ndef get_top_n_words(n, keys, document_term_matrix, count_vectorizer):\n    '''\n    returns a list of n_topic strings, where each string contains the n most common \n    words in a predicted category, in order\n    '''\n    top_word_indices = []\n    for topic in range(n_topics):\n        temp_vector_sum = 0\n        for i in range(len(keys)):\n            if keys[i] == topic:\n                temp_vector_sum += document_term_matrix[i]\n        temp_vector_sum = temp_vector_sum.toarray()\n        top_n_word_indices = np.flip(np.argsort(temp_vector_sum)[0][-n:],0)\n        top_word_indices.append(top_n_word_indices)   \n    top_words = []\n    for topic in top_word_indices:\n        topic_words = []\n        for index in topic:\n            temp_word_vector = np.zeros((1,document_term_matrix.shape[1]))\n            temp_word_vector[:,index] = 1\n            the_word = count_vectorizer.inverse_transform(temp_word_vector)[0][0]\n            topic_words.append(the_word.encode('ascii',errors=\"ignore\").decode('utf-8',errors=\"ignore\"))\n        top_words.append(\" \".join(topic_words))         \n    return top_words","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_n_words_lsa = get_top_n_words(10, lsa_keys, small_document_term_matrix, small_count_vectorizer)\n\nfor i in range(len(top_n_words_lsa)):\n    print(\"Topic {}: \".format(i+1), top_n_words_lsa[i])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Thus we have converted our initial small sample of headlines into a list of predicted topic categories, where each category is characterised by its most frequent words. The relative magnitudes of each of these categories can then be easily visualised though use of a bar chart."},{"metadata":{"trusted":true},"cell_type":"code","source":"top_3_words = get_top_n_words(3, lsa_keys, small_document_term_matrix, small_count_vectorizer)\nlabels = ['Topic {}: \\n'.format(i) + top_3_words[i] for i in lsa_categories]\n\nfig, ax = plt.subplots(figsize=(8,4))\nax.bar(lsa_categories, lsa_counts);\nax.set_xticks(lsa_categories);\nax.set_xticklabels(labels);\nax.set_ylabel('Number of comments');\nax.set_title('LSA topic counts');\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## However, this does not provide a great point of comparison with other clustering algorithms. In order to properly contrast LSA with LDA we instead use a dimensionality-reduction technique called  *t*-SNE, which will also serve to better illuminate the success of the clustering process."},{"metadata":{"trusted":true},"cell_type":"code","source":"tsne_lsa_model = TSNE(n_components=2, perplexity=50, learning_rate=100, \n                        n_iter=2000, verbose=1, random_state=0, angle=0.75)\ntsne_lsa_vectors = tsne_lsa_model.fit_transform(lsa_topic_matrix)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Now that we have reduced these n_topics-dimensional vectors to two-dimensional representations, we can then plot the clusters using Bokeh. Before doing so however, it will be useful to derive the centroid location of each topic, so as to better contextualise our visualisation."},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define helper functions\ndef get_mean_topic_vectors(keys, two_dim_vectors):\n    '''\n    returns a list of centroid vectors from each predicted topic category\n    '''\n    mean_topic_vectors = []\n    for t in range(n_topics):\n        articles_in_that_topic = []\n        for i in range(len(keys)):\n            if keys[i] == t:\n                articles_in_that_topic.append(two_dim_vectors[i])    \n        \n        articles_in_that_topic = np.vstack(articles_in_that_topic)\n        mean_article_in_that_topic = np.mean(articles_in_that_topic, axis=0)\n        mean_topic_vectors.append(mean_article_in_that_topic)\n    return mean_topic_vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"colormap = np.array([\n    \"#1f77b4\", \"#aec7e8\", \"#ff7f0e\", \"#ffbb78\", \"#2ca02c\",\n    \"#98df8a\", \"#d62728\", \"#ff9896\", \"#9467bd\", \"#c5b0d5\",\n    \"#8c564b\", \"#c49c94\", \"#e377c2\", \"#f7b6d2\", \"#7f7f7f\",\n    \"#c7c7c7\", \"#bcbd22\", \"#dbdb8d\", \"#17becf\", \"#9edae5\" ])\ncolormap = colormap[:n_topics]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## All that remains is to plot the clustered questions. Also included are the top three words in each cluster, which are placed"},{"metadata":{"trusted":true},"cell_type":"code","source":"top_3_words_lsa = get_top_n_words(3, lsa_keys, small_document_term_matrix, small_count_vectorizer)\nlsa_mean_topic_vectors = get_mean_topic_vectors(lsa_keys, tsne_lsa_vectors)\n\nplot = figure(title=\"t-SNE Clustering of {} LSA Topics\".format(n_topics), plot_width=700, plot_height=700)\nplot.scatter(x=tsne_lsa_vectors[:,0], y=tsne_lsa_vectors[:,1], color=colormap[lsa_keys])\n\nfor t in range(n_topics):\n    label = Label(x=lsa_mean_topic_vectors[t][0], y=lsa_mean_topic_vectors[t][1], \n                  text=top_3_words_lsa[t], text_color=colormap[t])\n    plot.add_layout(label)\n    \nshow(plot)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Evidently, this is a bit a of a failed result. We have failed to reach any great degree of separation across the topic categories, and it is difficult to tell whether this can be attributed to the LSA decomposition or instead the  t -SNE dimensionality reduction process. Let's move forward and try another clustering method"},{"metadata":{},"cell_type":"markdown","source":"# Latent Dirichilet Allocation (LDA)\n### We now repeat this process using LDA instead of LSA. LDA is instead a generative probabilistic process, designed with the specific goal of uncovering latent topic structure in text corpora."},{"metadata":{"trusted":true},"cell_type":"code","source":"lda_model = LatentDirichletAllocation(n_components=n_topics, learning_method='online', \n                                          random_state=0, verbose=0)\nlda_topic_matrix = lda_model.fit_transform(small_document_term_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lda_keys = get_keys(lda_topic_matrix)\nlda_categories, lda_counts = keys_to_counts(lda_keys)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_n_words_lda = get_top_n_words(10, lda_keys, small_document_term_matrix, small_count_vectorizer)\n\nfor i in range(len(top_n_words_lda)):\n    print(\"Topic {}: \".format(i+1), top_n_words_lda[i])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_3_words = get_top_n_words(3, lda_keys, small_document_term_matrix, small_count_vectorizer)\nlabels = ['Topic {}: \\n'.format(i) + top_3_words[i] for i in lda_categories]\n\nfig, ax = plt.subplots(figsize=(10,4))\nax.bar(lda_categories, lda_counts);\nax.set_xticks(lda_categories);\nax.set_xticklabels(labels);\nax.set_title('LDA topic counts');\nax.set_ylabel('Number of comments');","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## However, in order to properly compare LDA with LSA, we again take this topic matrix and project it into two dimensions."},{"metadata":{"trusted":true},"cell_type":"code","source":"tsne_lda_model = TSNE(n_components=2, perplexity=50, learning_rate=100, \n                        n_iter=2000, verbose=1, random_state=0, angle=0.75)\ntsne_lda_vectors = tsne_lda_model.fit_transform(lda_topic_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_3_words_lda = get_top_n_words(3, lda_keys, small_document_term_matrix, small_count_vectorizer)\nlda_mean_topic_vectors = get_mean_topic_vectors(lda_keys, tsne_lda_vectors)\n\nplot = figure(title=\"t-SNE Clustering of {} LDA Topics\".format(n_topics), plot_width=600, plot_height=600)\nplot.scatter(x=tsne_lda_vectors[:,0], y=tsne_lda_vectors[:,1], color=colormap[lda_keys])\n\nfor t in range(n_topics):\n    label = Label(x=lda_mean_topic_vectors[t][0], y=lda_mean_topic_vectors[t][1], \n                  text=top_3_words_lda[t], text_color=colormap[t])\n    plot.add_layout(label)\n\nshow(plot)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### This is a much better result! Controlling for  t -SNE, it would seem that LDA has had much more succcess than LSA in separating out the topic categories. For this reason, LDA appears the more appropriate algorithm when we scale up the clustering process.\n# LDA with an other way of visualisation"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preparing a corpus for analysis and checking the first 5 entries\ncorpus=[]\n\ncorpus = df_train['comment_text'].to_list()\n\ncorpus[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEMP_FOLDER = tempfile.gettempdir()\nprint('Folder \"{}\" will be used to save temporary dictionary and corpus.'.format(TEMP_FOLDER))\n\nlogging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# removing common words and tokenizing\n# google-quest-challenge\nstoplist = stopwords.words('english') + list(punctuation) + list(\"([)]?\") + [\")?\"]\n\ntexts = [[word for word in str(document).lower().split() if word not in stoplist] for document in corpus]\n\ndictionary = corpora.Dictionary(texts)\ndictionary.save(os.path.join(TEMP_FOLDER, 'tweets.dict'))  # store the dictionary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corpus = [dictionary.doc2bow(text) for text in texts]\ncorpora.MmCorpus.serialize(os.path.join(TEMP_FOLDER, 'tweets.mm'), corpus) ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## In the previous cells, I created a corpus of documents represented as a stream of vectors. To continue, lets use that corpus, with the help of Gensim.\n\n#### Gensim is an open-source library for unsupervised topic modeling and natural language processing, using modern statistical machine learning. Gensim is implemented in Python and Cython. Wikipedia\n# Creating a transformation\n### The transformations are standard Python objects, typically initialized by means of a training corpus:\n\n### Different transformations may require different initialization parameters; in case of TfIdf, the “training” consists simply of going through the supplied corpus once and computing document frequencies of all its features. Training other models, such as Latent Semantic Analysis or Latent Dirichlet Allocation"},{"metadata":{"trusted":true},"cell_type":"code","source":"tfidf = models.TfidfModel(corpus) # step 1 -- initialize a model\ncorpus_tfidf = tfidf[corpus]  # step 2 -- use the model to transform vectors","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LDA:\n\n### Latent Dirichlet Allocation, LDA is yet another transformation from bag-of-words counts into a topic space of lower dimensionality. LDA is a probabilistic extension of LSA (also called multinomial PCA), so LDA’s topics can be interpreted as probability distributions over words. These distributions are, just like with LSA, inferred automatically from a training corpus. Documents are in turn interpreted as a (soft) mixture of these topics (again, just like with LSA)."},{"metadata":{"trusted":true},"cell_type":"code","source":"#I will try 10 topics\ntotal_topics = 5\n\nlda = models.LdaModel(corpus, id2word=dictionary, num_topics=total_topics)\ncorpus_lda = lda[corpus_tfidf] # create a double wrapper over the original corpus: bow->tf","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Show first n important word in the topics"},{"metadata":{},"cell_type":"markdown","source":"lda.show_topics(total_topics,5)"},{"metadata":{"trusted":true},"cell_type":"code","source":"data_lda = {i: OrderedDict(lda.show_topic(i,25)) for i in range(total_topics)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_lda = pd.DataFrame(data_lda)\ndf_lda = df_lda.fillna(0).T\nprint(df_lda.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_lda","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"g=sns.clustermap(df_lda.corr(), center=0, standard_scale=1, cmap=\"OrRd\", metric='cosine', linewidths=.75, figsize=(12, 12))\nplt.setp(g.ax_heatmap.yaxis.get_majorticklabels(), rotation=0)\nplt.show()\n#plt.setp(ax_heatmap.get_yticklabels(), rotation=0)  # For y axis","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Please don't forget to upvote!\n\n# To be continued"}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}