{"cells":[{"metadata":{},"cell_type":"markdown","source":"### This Notebook is an exploration of the QA dataset, especially to cluster similar questions.\n\n\n### Clustering is similar to that of document categorization, where you start with a whole corpus of documents and are tasked with segregating them into various groups based on some distinctive properties, attributes, and features of the documents.\n\n### We will try three different clustering algorithms in this notebook:\n\n    * K-means clustering\n    * Affinity propagation\n    * Ward’s agglomerative hierarchical clustering"},{"metadata":{},"cell_type":"markdown","source":"### Importing libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\nimport numpy as np \nimport pandas as pd\nimport re\nimport os\nimport random\n\n# For plotting\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport seaborn as sns\nsns.set(style='white', context='notebook', rc={'figure.figsize':(14,10)})\n\nfrom tqdm import tqdm_notebook as tqdm\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction import text\nfrom sklearn.metrics.pairwise import cosine_similarity\n\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))        ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Reading the JSON files"},{"metadata":{"trusted":true},"cell_type":"code","source":"ids = []\nans = []\ncandidates = []\nquestions = []\n\nwith open('/kaggle/input/tensorflow2-question-answering/simplified-nq-train.jsonl', 'r') as json_file:\n    cnt = 0\n    for line in tqdm(json_file):\n        json_data = json.loads(line)        \n        ids.append(str(json_data['example_id']))\n        questions.append(json_data['question_text'])\n        candidates = json_data['long_answer_candidates']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_data = pd.DataFrame()\n\ntr_data['example_id'] = ids\ntr_data['question'] = questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_data.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Normalization and Feature extraction "},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.stem.lancaster import LancasterStemmer\nfrom nltk.stem.porter import *\nimport nltk\nfrom nltk.util import ngrams\nfrom nltk.corpus import stopwords\nfrom nltk import sent_tokenize, word_tokenize\nimport re\nimport string\nfrom nltk.stem import WordNetLemmatizer\n\nstopword_list = nltk.corpus.stopwords.words('english')\nwnl = WordNetLemmatizer()\nps = PorterStemmer()\n\ntokenizer = nltk.RegexpTokenizer(r'\\w+')\nstop_words = set(stopwords.words('english'))\n\ndef preprocessing(data):\n    txt = data.str.lower().str.cat(sep=' ')\n    words = tokenizer.tokenize(txt)\n    words = [w for w in words if not w in stop_words]\n    return words\n\ndef tokenize_text(text):\n    tokens = nltk.word_tokenize(text)\n    tokens = [token.strip() for token in tokens]\n    return tokens","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_special_characters(text):\n    tokens = tokenize_text(text)\n    pattern = re.compile('[{}]'.format(re.escape(string.punctuation)))\n    filtered_tokens = filter(None, [pattern.sub('', token) for token in tokens])\n    filtered_text = ' '.join(filtered_tokens)\n    return filtered_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_stopwords(text):\n    tokens = tokenize_text(text)\n    filtered_tokens = [token for token in tokens if token not in stopword_list]\n    filtered_text = ' '.join(filtered_tokens)\n    return filtered_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def keep_text_characters(text):\n    filtered_tokens = []\n    tokens = tokenize_text(text)\n    for token in tokens:\n        if re.search('[a-zA-Z]', token):\n            filtered_tokens.append(token)\n    filtered_text = ' '.join(filtered_tokens)\n    return filtered_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def normalize_corpus(corpus, lemmatize=True,  only_text_chars=False, tokenize=False):\n    normalized_corpus = []\n    for text in corpus:\n        text = text.lower()\n        text = remove_special_characters(text)\n        text = remove_stopwords(text)\n        if only_text_chars:\n            text = keep_text_characters(text)\n \n        if tokenize:\n            text = tokenize_text(text)\n            normalized_corpus.append(text)\n        else:\n            normalized_corpus.append(text)\n    return normalized_corpus","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n\ndef build_feature_matrix(documents, feature_type='frequency',  ngram_range=(1, 1), min_df=0.0, max_df=1.0):\n    feature_type = feature_type.lower().strip()\n    if feature_type == 'binary':\n        vectorizer = CountVectorizer(binary=True, min_df=min_df, max_df=max_df, ngram_range=ngram_range)\n    elif feature_type == 'frequency':\n        vectorizer = CountVectorizer(binary=False, min_df=min_df, max_df=max_df, ngram_range=ngram_range)\n    elif feature_type == 'tfidf':\n        vectorizer = TfidfVectorizer(min_df=min_df, max_df=max_df, ngram_range=ngram_range)\n    else:\n        raise Exception(\"Wrong feature type. Possible values are binary, frequency, or tfidf\")\n    feature_matrix = vectorizer.fit_transform(documents).astype(float)\n    return vectorizer, feature_matrix","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Normalize Corpus"},{"metadata":{"trusted":true},"cell_type":"code","source":"## Taking only a subset of questions\n\nqns = tr_data['question'][:500]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"norm_docs = normalize_corpus(qns,  lemmatize=True, only_text_chars=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tr_data_500 = tr_data.iloc[:500]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Extract features"},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer, feature_matrix = build_feature_matrix(norm_docs, feature_type='tfidf', min_df=0, max_df=0.8, ngram_range=(1, 1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Viewing the number of features & getting the feature names"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(feature_matrix.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# get feature names\nfeature_names = vectorizer.get_feature_names()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# print sample features\nprint(feature_names[:20])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Above are some of the features extracted from the normalized documents"},{"metadata":{},"cell_type":"markdown","source":"## K-means Clustering\n\nThe k-means clustering algorithm is a centroid-based clustering model that tries to cluster data into groups or clusters of equal variance.\nThe criteria or measure that this algorithm tries to minimize is inertia, also known as within-cluster sum-of-squares. One main disadvantage of this algorithm is that the number of clusters k need to be specified in advance, as is the case with all other centroid-based clustering models."},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.cluster import KMeans\n\n# define the k-means clustering function\n\ndef k_means(feature_matrix, num_clusters=5):\n    km = KMeans(n_clusters=num_clusters, max_iter=10000)\n    km.fit(feature_matrix)\n    clusters = km.labels_\n    return km, clusters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# set k = 10(decided arbitrarily, right approach would be elbow method/silhoutte score which we will get to).\n# Lets say we want 10 clusters from the list of questions we got \n\nnum_clusters = 10\nkm_obj, clusters = k_means(feature_matrix=feature_matrix,num_clusters=num_clusters)\ntr_data_500['Clusters'] = clusters","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\n\n## Getting the total questions per cluster \n\nc = Counter(clusters)\nprint(c.items())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Since we have not used any word embedding techniques to extract features, what we have now won't be the best clustering model. We will start by defining a function to extract important information from our cluster analysis:"},{"metadata":{},"cell_type":"markdown","source":"The below function is pretty self-explanatory. What it does is basically extract the key features per cluster that were essential in defining the cluster    from the centroids. It also  retrieves the questions' example ID that belong to each cluster and stores everything in a dictionary.\nWe will now define a function that uses this data structure and prints the results in a clear format:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_cluster_data(clustering_obj, tr_data_500, feature_names, num_clusters,topn_features=10):\n    cluster_details = {}\n    # get cluster centroids\n    ordered_centroids = clustering_obj.cluster_centers_.argsort()[:, ::-1]\n    # get key features & questions for each cluster\n    \n    for cluster_num in range(num_clusters):\n        cluster_details[cluster_num] = {}\n        cluster_details[cluster_num]['cluster_num'] = cluster_num\n        key_features = [feature_names[index] for index in ordered_centroids[cluster_num, :topn_features]]\n        cluster_details[cluster_num]['key_features'] = key_features\n        qnss = tr_data_500[tr_data_500['Clusters'] == cluster_num]['example_id'].values.tolist()\n        cluster_details[cluster_num]['Questions'] = qnss\n    return cluster_details","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def print_cluster_data(cluster_data):\n    # print cluster details\n    for cluster_num, cluster_details in cluster_data.items():\n        print('Cluster {} details:'.format(cluster_num))\n        print('-'*20)\n        print('Key features:', cluster_details['key_features'])\n        print(\"Example ID's in this cluster:\")\n        print(', '.join(cluster_details['Questions']))\n        print('='*80)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get clustering analysis data\n\ncluster_data = get_cluster_data(clustering_obj=km_obj, tr_data_500=tr_data_500, feature_names=feature_names, num_clusters=num_clusters, topn_features=5)\n\n# print clustering analysis results to see what are those features that come under the same cluster\n\nprint_cluster_data(cluster_data)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualize the clusters\n\n\nThere are challenges associated with visualizing clusters. This happens especially when dealing with multidimensional feature spaces and unstructured text data, such as this dataset. Dimensionality reduction techniques can be applied here to reduce the dimensionality such that we can visualize these clusters in 2- or 3-dimensional plots. We will be using PCA here for visualizing clusters."},{"metadata":{"trusted":true},"cell_type":"code","source":"## Importing and Apply PCA\n\ncosine_distance = 1 - cosine_similarity(feature_matrix)\n\nfrom sklearn.decomposition import PCA\n\npca = PCA(n_components=2) # project from 784 to 2 dimensions\n\nprincipalComponents = pca.fit_transform(cosine_distance)\n\np_df = pd.DataFrame(data = principalComponents, columns = ['principal component 1', 'principal component 2'])\n\np_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Explaining the Variance ratio\n\nprint('Explained variation per principal component: {}'.format(pca.explained_variance_ratio_))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the first two principal components of each point to learn about the data:\n\nfrom pylab import rcParams\nrcParams['figure.figsize'] = 17, 9\n\nplt.scatter(principalComponents[:, 0], principalComponents[:, 1], s= 5, c=clusters, cmap='Spectral')\n\nplt.gca().set_aspect('equal', 'datalim')\n\nplt.colorbar(boundaries=np.arange(11)-0.5).set_ticks(np.arange(10))\n\nplt.title('Visualizing the clusters', fontsize=25);\n\nplt.xlabel('Principal Component 1')\nplt.ylabel('Principal Component 2')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### What can be seen in the plot above is there are cluster of documents(questions) that are close to each other(purple ones mainly) and there are questions that are under the same cluster but distance apart(cosine distance). "},{"metadata":{},"cell_type":"markdown","source":"### Other 2 clustering techniques + Embeddings to follow, Meanwhile share your thoughts and hey upvote? :)"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}