{"cells":[{"metadata":{"_uuid":"47f7b2dbc73bcb742b870c24e1ec3504a6b89237"},"cell_type":"markdown","source":"  In this Notebook we will look into some interesting facts regarding question distributions, as never seen before !! :-P \n \n1. Firstly we will perform some **data cleaning** and **preprocessing**\n * Tokenization of questions using regular exp tokenizer\n * Removal of stop words\n1.  Then we produce sentence mean vectors for each of the question, by first mapping the words to their word embedding (*Google News Dataset*), and then we compute the mean of the word embeddings to generate sentence embedding\n1.  We then visualize these sentence embeddings in 3D after reducing their dimensions from 300d to 3d using **Principal Component Analysis**\n1. We then analyze how close the sincere and insincere questions are to topics like **sexism** and **racism**  and visualize the distribution using cosine similarity\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"#All necessary packages imported here\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport nltk\nfrom nltk.tokenize import RegexpTokenizer\nfrom nltk.corpus import stopwords\nstop_words = set(stopwords.words('english'))\nimport gc\nimport re\nimport gensim\n\nprint(os.listdir(\"../input\"))\ndf=pd.read_csv('../input/train.csv')\nprint (df.columns)\nprint (df.question_text.describe())\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ffd4c607bd4721214b2d11b5fd71e1e45ab9c513","_kg_hide-input":false},"cell_type":"code","source":"# Loading the Google News Word2Vec Model\nmodel = gensim.models.KeyedVectors.load_word2vec_format('../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin', binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4bdcae2ce5a0bb4d7de8cd1c5d313084b1228bbe"},"cell_type":"code","source":"tokenizer=RegexpTokenizer(r'\\w+')\n#Tokenizing Questions\ndf['tokenized_text']=[tokenizer.tokenize(i) for i in df.question_text]\n#Removal of Stop Words from tokenized questions\ntemp=list(list(map(lambda x:x if x not in stop_words else False,df.tokenized_text[i])) for i in range(len(df)))\ndf['tokenized_text']=list(list(filter(lambda x: x is not False,temp[i]))for i in range(len(temp)))\ndel temp\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"3746110be796b71d9d6c0796d6da6220251ee879","_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"#Mapping all tokenized words to embeddings\nwv=[]\ndef assign_word_embed(word):\n    try:\n        return model[word]\n    except:\n        pass\n    \nprint(df['tokenized_text'][0][0])\ndf['sentence_vectors']=list(list(map(lambda x:assign_word_embed(x),df['tokenized_text'][i])) for i in range(len(df)))\n# temp2=list(map(lambda x:assign_word_embed(x),df['tokenized_text'][0][0]))\nprint (len(df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"97372efeb8accc14c732719dfda662d79afcca52","_kg_hide-output":false},"cell_type":"code","source":"#Creating sentence embedding as mean of word embeddings of the words in sentence\nfrom statistics import mean\ndef mean_comp(lst):\n    length=len(lst)\n    sum=np.zeros(300)\n    for i in lst:\n        try:\n            sum+=i\n        except:\n            \"\"\"When value is None because word \n            wasn't present in the Google News vocabulary\"\"\"\n            length-=1\n    if length==0:\n        return np.zeros(300)\n    else:\n        return (sum/length)\n #Sample Data for Visualization\ndf['mean_vectors']=[(mean_comp(df['sentence_vectors'][i])) for i in range(len(df))] \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7ab32349006cfaa603f043f5c4c23b3967d0bfd5","scrolled":false,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"# temp=[(df['mean_vectors'][i]) for i in range(len(df))]\nprint((df.size))\ndf=df.drop('question_text',axis=1)\ndf=df.drop('tokenized_text',axis=1)\nprint((df.size))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"abdd4566f30750743b295ab4a19d2ca06566ea78","scrolled":false},"cell_type":"code","source":"#Dimensionality reduction using PCA and Visualization the two types of questions 3D\nfrom sklearn.decomposition import PCA\npca = PCA(n_components=3)\na=[[1,1,1],[2,2,2],[3,3,3]]\ntemp= [df['mean_vectors'][i]for i in range(500000)]\nprint (type(temp[0]))\npca_result = pca.fit_transform(temp)\npca_first_comp=[i[0]for i in pca_result]\npca_second_comp=[i[1]for i in pca_result]\npca_third_comp=[i[2]for i in pca_result]\nprint(pca.explained_variance_ratio_)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4d49677dd06017bb50ce96555120c1be00e4b2f3","_kg_hide-input":true},"cell_type":"code","source":"from matplotlib import pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"074b079cb53b257f30e678657a881906de2895a7","_kg_hide-output":false},"cell_type":"code","source":"from mpl_toolkits.mplot3d import Axes3D\ncolor=['red'if l == 0 else 'blue' for l in df['target'][0:500000]]\nfig=plt.figure()\nax = fig.gca(projection='3d')\nax=plt.scatter(pca_first_comp,pca_second_comp,pca_third_comp,c=color)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc80db1e677b7306388fd8db84f2036b5d779893"},"cell_type":"code","source":"#Checking how the data is clustered into discriminative and sexuality based sentences\nwv_sex=(model['sex']+model['incest']+model['bestiality']+model['pedophilia'])/4\n# wv_racism=\nwv_discrimination=(model['race']+model['color']+model['caste'])/3\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7eb8f01f3e7b1faa5085b46f47295e764ffba69a","_kg_hide-output":true},"cell_type":"code","source":"#Performing Cosine Similarity \nfrom sklearn.metrics.pairwise import cosine_similarity as cs\nfrom scipy import spatial\nresult=1-spatial.distance.cosine(df['mean_vectors'][0],wv_sex)\ncs_sex=list(map(lambda x:1-spatial.distance.cosine(x,wv_sex),df['mean_vectors']))\n# cs_racism=list(map(lambda x:1-spatial.distance.cosine(x,wv_sex),temp2))\ncs_discrimination=list(map(lambda x:1-spatial.distance.cosine(x,wv_discrimination),df['mean_vectors']))\n# cs_sex=[cs(i.reshape(1,-1),wv_sex.reshape(1,-1)) for i in temp2]\nprint(cs_sex[0],result)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fb5ed9b13ec7daef1d03be834e67169b82a6bc5a"},"cell_type":"code","source":"sincere_questions={}\ninsincere_questions={}\nsincere_questions['discrimination']=[cs_discrimination[i] for i in range(len(cs_discrimination)) if  df['target'][i]==0]\ninsincere_questions['discrimination']=[cs_discrimination[i] for i in range(len(cs_discrimination)) if df['target'][i]==1]\nsincere_questions['sex']=[cs_sex[i] for i in range(len(cs_sex)) if  df['target'][i]==0]\ninsincere_questions['sex']=[cs_sex[i] for i in range(len(cs_sex)) if df['target'][i]==1]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6ca113adccc3a16d58baeb269081883b9e9f828f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"662eb34a012cb4e118f4511008b528cf21bc3232","scrolled":true},"cell_type":"code","source":"#Plots to Visualize how the questions are distributed with respect to their distance from sexuality and discrimination \nfig = plt.figure()\nfig, ((ax1, ax2), (ax3, ax4)) = plt.subplots(nrows=2, ncols=2)\nplt.tight_layout()\nax1.hist([insincere_questions['discrimination'][0:1200]],bins=100)\nax3.hist([sincere_questions['discrimination'][0:1200]],bins=100)\nax1.set_title(\"insincere discriminative\")\nax2.set_title(\"insincere sexual\")\nax3.set_title(\"sincere discriminative\")\nax4.set_title(\"sincere sexual\")\nax2.hist([insincere_questions['sex'][0:1200]],bins=100)\nax4.hist([sincere_questions['sex'][0:1200]],bins=100)\n# print (insincere_questions)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab84157db218e41d48acb172afa295f2351a2ce8"},"cell_type":"markdown","source":"It is clearly observed how the Insincere Questions are biased a lil more to the right than the Sincere Questions(higher similarity to the topics such as **sexuality** and **discrimination**)"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}