{"cells":[{"metadata":{},"cell_type":"markdown","source":"# 1. Data import"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # data visiualization\nimport seaborn as sns # data visisualization like distribytion chart, matrix plot, heat maps\nimport sklearn # scikit library for machine learning\n\n###########","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom sklearn import preprocessing,metrics,manifold\nfrom sklearn.manifold import TSNE\nfrom sklearn.model_selection import train_test_split,cross_val_score,GridSearchCV,cross_val_predict\nfrom imblearn.over_sampling import ADASYN,SMOTE\nfrom imblearn.under_sampling import NearMiss\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.svm import SVC\nimport collections\nimport matplotlib.patches as mpatches\nfrom sklearn.metrics import accuracy_score\n%matplotlib inline\nfrom sklearn.preprocessing import RobustScaler\nimport xgboost\nfrom imblearn.metrics import classification_report_imbalanced\nfrom sklearn.metrics import classification_report,roc_auc_score,roc_curve,r2_score,recall_score,confusion_matrix,precision_recall_curve\nfrom collections import Counter\nfrom sklearn.model_selection import StratifiedKFold,KFold,StratifiedShuffleSplit\nfrom nltk import word_tokenize\nfrom nltk.corpus import stopwords\nstop_words = stopwords.words('english')\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.decomposition import PCA, TruncatedSVD,SparsePCA\nfrom sklearn.metrics import classification_report,confusion_matrix\nfrom nltk.tokenize import word_tokenize\nfrom collections import defaultdict\nfrom collections import Counter\nimport seaborn as sns\nfrom wordcloud import WordCloud,STOPWORDS\nimport nltk\nfrom nltk.corpus import stopwords\nimport string\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\nsample_submission=pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"   # 2.Descriptives"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.columns)\nprint(train.info)\nprint(train.head())\nprint(train.describe())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Exploratory data analysis"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train['question_text'].str.len().hist() #number of characters present in each sentence","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Number of words appearing in each line."},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train['question_text'].str.split().map(lambda x: len(x)).hist()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Average word length in each sentence"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train['question_text'].str.split().apply(lambda x : [len(i) for i in x]). map(lambda x: np.mean(x)).hist() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Measuring insincere and sincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"insincere=train[train['target']==1]\n\nsincere=train[train['target']==0]\n\nprint(\"First 10 samples of insincere questions\\n\".format(),insincere[:10])\n\nprint(\"First 10 samples of sincere \\n\".format(),sincere[:10])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Frequency count of sincere and insincere question texts:"},{"metadata":{"trusted":true},"cell_type":"code","source":"count=train['target'].value_counts()\n\nprint('Total Counts of both sets'.format(),count)\n\nprint(\"==============\")\n\n#Creating a function to plot the counts using matplotlib\ndef plot_counts(count_good,count_bad):\n    plt.rcParams['figure.figsize']=(6,6)\n    plt.bar(0,count_good,width=0.6,\n            label='Sincere Questions',\n            color='Indigo')\n    plt.legend()\n    plt.bar(2,count_bad,width=0.6,\n            label='Insincere Questions',\n            color='Red')\n    plt.legend()\n    plt.ylabel('Count of Questions')\n    plt.xlabel('Types of Questions')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_good=train[train['target']==0]\ncount_bad=train[train['target']==1]\nplot_counts(len(count_good),len(count_bad))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Analyse the count of words in each segment- both positive and negative reviews"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Function for checking word length\ndef cal_len(data):\n    return len(data)\n\n#Create generic plotter with Seaborn\ndef plot_count(count_ones,count_zeros,title_1,title_2,subtitle):\n    fig,(ax1,ax2)=plt.subplots(1,2,figsize=(15,5))\n    sns.distplot(count_zeros,ax=ax1,color='Blue')\n    ax1.set_title(title_1)\n    sns.distplot(count_ones,ax=ax2,color='Red')\n    ax2.set_title(title_2)\n    fig.suptitle(subtitle)\n    plt.show()   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_sincere=count_good['question_text'].str.split().apply(lambda z:cal_len(z))\ncount_insincere=count_bad['question_text'].str.split().apply(lambda z:cal_len(z))\nprint(\"Sincere Questions:\" + str(count_sincere))\nprint(\"Insincere Questions:\" + str(count_insincere))\nplot_count(count_sincere,count_insincere,\"Sincere Questions\",\n           \"Insincere Questions\",\"Questions Analysis\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Count Punctuations/Stopwords/Codes and other semantic datatypes"},{"metadata":{"trusted":true},"cell_type":"code","source":"#We will be using the \"generic_plotter\" function.\n\ncount_sincere_punctuations=count_good['question_text'].apply(lambda z: len([c for c in str(z) if c in string.punctuation]))\n\ncount_insincere_punctuations=count_bad['question_text'].apply(lambda z:len([c for c in str(z) if c in string.punctuation]))\n\nplot_count(count_sincere_punctuations,count_insincere_punctuations,\"Sincere Questions Punctuations\",\"Insincere Questions Punctuations\",\"Questions Punctuation Analysis\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Analyse Stopwords"},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_count_1(count_ones,count_zeros,title_1,title_2,subtitle):\n    fig,(ax1,ax2)=plt.subplots(1,2,figsize=(15,5))\n    sns.distplot(count_zeros,ax=ax1,color='Blue')\n    ax1.set_title(title_1)\n    sns.distplot(count_ones,ax=ax2,color='Orange')\n    ax2.set_title(title_2)\n    fig.suptitle(subtitle)\n    plt.show()    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stops=set(stopwords.words('english'))\n\ncount_sincere_stops=count_good['question_text'].apply(lambda z : np.mean([len(z) for w in str(z).split()]))\n\ncount_insincere_stops=count_bad['question_text'].apply(lambda z : np.mean([len(z) for w in str(z).split()]))\n\nplot_count_1(count_sincere_stops,count_insincere_stops,\"Sincere Questions Stopwords\",\"Insincere Questions Stopwords\",\"Questions Stopwords Analysis\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization of most occuring words through word cloud"},{"metadata":{"trusted":true},"cell_type":"code","source":"def display_cloud(data,color,figsize):\n    plt.subplots(figsize=figsize)\n    wc = WordCloud(stopwords=STOPWORDS,\n                   background_color=\"white\", \n                   contour_width=2, \n                   contour_color=color,\n                   max_words=2000, \n                   max_font_size=256,\n                   random_state=42)\n    wc.generate(' '.join(data))\n    plt.imshow(wc, interpolation=\"bilinear\")\n    plt.axis('off')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_cloud(train['question_text'],\n              color='red',\n              figsize=(5,5)) #WordCloud for the insincere questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_cloud(sincere['question_text'],\n              color='red',\n              figsize=(5,5)) #WordCloud for the sincere questions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_cloud(insincere['question_text'],\n              color='red',\n              figsize=(5,5)) #WordCloud for the insincere questions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building corpus"},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\nnltk.download('stopwords')\nstop=set(stopwords.words('english'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#create the corpus.\n\ncorpus=[]\nquestion_text = train['question_text'].str.split()\nquestion_text= question_text.values.tolist()\ncorpus=[word for i in question_text for word in i]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import defaultdict\ndic=defaultdict(int)\nfor word in corpus:\n    if word in stop:\n        dic[word]+=1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## occurrences of each word in a list of tuples"},{"metadata":{"trusted":true},"cell_type":"code","source":"counter= Counter(corpus)\nmost = counter.most_common()\n\nx, y= [], []\nfor word,count in most[:40]:\n    if (word not in stop):\n        x.append(word)\n        y.append(count)\n        \nsns.barplot(x=y,y=x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Inference From Exploratory Analysis\n\nThe following can be inferred from the data:\n\nThe dataset is imbalanced.\nThe dataset contains redundant words and html syntaxes.\nPunctuations/stopwords are present in a equal distribution in the dataset.\nThis tells us that we have to do lots of cleaning!"},{"metadata":{},"cell_type":"markdown","source":"## N-grams"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"#Ngram exploration\nfrom nltk.util import ngrams","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def get_top_ngram(corpus, n=None):\n    vec = CountVectorizer(ngram_range=(n, n)).fit(corpus)\n    bag_of_words = vec.transform(corpus)\n    sum_words = bag_of_words.sum(axis=0) \n    words_freq = [(word, sum_words[0, idx]) \n                  for word, idx in vec.vocabulary_.items()]\n    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)\n    return words_freq[:10]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## bigrams"},{"metadata":{"trusted":true},"cell_type":"code","source":"insincere_label=train[train['target']== 1]['question_text']\nprint(insincere_label.head())  #top insincere questions\n\n\nsincere_label=train[train['target']== 0]['question_text']\nprint(sincere_label.head())  #top sincere questions","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Bigram Insincere Questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"bigram_insincere=get_top_ngram(insincere_label,n=2)\nx,y=map(list,zip(*bigram_insincere))\nsns.barplot(x=y,y=x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Bigram sincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"bigram_sincere=get_top_ngram(sincere_label,n=2)\nx,y=map(list,zip(*bigram_sincere))\nsns.barplot(x=y,y=x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Trigrams"},{"metadata":{},"cell_type":"markdown","source":"Trigram insincere questions"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"trigram_insincere=get_top_ngram(insincere_label,n=3)\nx,y=map(list,zip(*trigram_insincere))\nsns.barplot(x=y,y=x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Trigram sincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"trigram_sincere=get_top_ngram(sincere_label,n=3)\nx,y=map(list,zip(*trigram_sincere))\nsns.barplot(x=y,y=x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Pentagram"},{"metadata":{},"cell_type":"markdown","source":"Pentagram insincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"pentagram_insincere=get_top_ngram(insincere_label,n=5)\nx,y=map(list,zip(*pentagram_insincere))\nsns.barplot(x=y,y=x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Pentagram sincere questions"},{"metadata":{"trusted":true},"cell_type":"code","source":"pentagram_sincere=get_top_ngram(sincere_label,n=5)\nx,y=map(list,zip(*pentagram_sincere))\nsns.barplot(x=y,y=x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Cleaning\n\nGetting rid of \n1. HTML codes\n2. URLs\n3. Emojis\n4. Stopwords\n5. Punctuations\n6. Expanding Abbreviations"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nimport re\n#Removes Punctuations\ndef remove_punctuations(data):\n    punct_tag=re.compile(r'[^\\w\\s]')\n    data=punct_tag.sub(r'',data)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['question_text']=train['question_text'].apply(lambda z: remove_punctuations(z))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Removes HTML syntaxes\ndef remove_html(data):\n    html_tag=re.compile(r'<.*?>')\n    data=html_tag.sub(r'',data)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['question_text']=train['question_text'].apply(lambda z: remove_html(z))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Removes URL data\ndef remove_url(data):\n    url_clean= re.compile(r\"https://\\S+|www\\.\\S+\")\n    data=url_clean.sub(r'',data)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['question_text']=train['question_text'].apply(lambda z: remove_url(z))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Removes Emojis\ndef remove_emoji(data):\n    emoji_clean= re.compile(\"[\"\n                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                           u\"\\U00002702-\\U000027B0\"\n                           u\"\\U000024C2-\\U0001F251\"\n                           \"]+\", flags=re.UNICODE)\n    data=emoji_clean.sub(r'',data)\n    url_clean= re.compile(r\"https://\\S+|www\\.\\S+\")\n    data=url_clean.sub(r'',data)\n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['question_text']=train['question_text'].apply(lambda z: remove_emoji(z))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['question_text'][:5]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Transforming corpus"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Lemmatize the dataset\nfrom nltk.stem import WordNetLemmatizer\n\n\ndef lemma_traincorpus(data):\n    lemmatizer=WordNetLemmatizer()\n    out_data=\"\"\n    for words in data:\n        out_data+= lemmatizer.lemmatize(words)\n    return out_data\n\ntrain['question_text']=train['question_text'].apply(lambda z: lemma_traincorpus(z))\n\ntrain['question_text'].sample()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.stem import *\n\ndef stem_traincorpus(data):\n    stemmer = PorterStemmer()\n    out_data=\"\"\n    for words in data:\n        out_data+= stemmer.stem(words)\n    return out_data\n\ntrain['question_text']=train['question_text'].apply(lambda z: stem_traincorpus(z))\n\ntrain['question_text'].sample()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Vectorization and Embeddings"},{"metadata":{},"cell_type":"markdown","source":"## TF-IDF"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntfidf_vect=TfidfVectorizer(stop_words='english',\n                           ngram_range=(1,3))\n\ntrain_tfidf=tfidf_vect.fit_transform(train['question_text'].values.tolist())\n\ntrain_tfidf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train_tfidf)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Visualizing the Vector Space"},{"metadata":{"trusted":true},"cell_type":"code","source":"from collections import Counter\nfrom tqdm import tqdm\nfrom scipy.sparse import csr_matrix\nimport math\nimport operator\nfrom sklearn.preprocessing import normalize\nimport numpy as np ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Count Vectorization\nimport matplotlib\nimport matplotlib.pyplot as plt\n\ndef vectorize(data):\n    cv=CountVectorizer()\n    fit_data_cv=cv.fit_transform(data)\n    return fit_data_cv,cv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Tfidf vectorization from sklearn\ndef tfidf(data):\n    tfidfv=TfidfVectorizer()\n    fit_data_tfidf=tfidfv.fit_transform(data)\n    return fit_data_cv,tfidfv","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dimensionality reduction"},{"metadata":{"trusted":true},"cell_type":"code","source":"def dimen_reduc_plot(test_data,\n                     test_label,\n                     option):\n    tsvd= TruncatedSVD(n_components=2,\n                       algorithm=\"randomized\",\n                       random_state=42)\n    tsne=TSNE(n_components=2,\n              random_state=42) #not recommended instead use PCA\n    pca=SparsePCA(n_components=2,\n                  random_state=42)\n    if(option==1):\n        tsvd_result=tsvd.fit_transform(test_data)\n        plt.figure(figsize=(10,8))\n        colors=['orange','red']\n        \n        sns.scatterplot(x=tsvd_result[:,0],\n                        y=tsvd_result[:,1],\n                        hue=test_label        )\n        \n        plt.show()\n        plt.figure(figsize=(10,10))\n        plt.scatter(tsvd_result[:,0],\n                    tsvd_result[:,1],\n                    c=test_label,\n                    cmap=matplotlib.colors.ListedColormap(colors))\n        color_red=mpatches.Patch(color='red',\n                                 label='Insincere Questions')\n        color_orange=mpatches.Patch(color='orange',\n                                    label='Sincere Questions')\n        plt.legend(handles=[color_orange,\n                            color_red])\n        plt.title(\"TSVD\")\n        plt.show()\n    if(option==2):\n        tsne_result=tsne.fit_transform(test_data)\n        plt.figure(figsize=(10,8))\n        colors=['orange','red']\n        sns.scatterplot(x=tsne_result[:,0],\n                        y=tsne_result[:,1],\n                        hue=test_label)\n        plt.show()\n        plt.figure(figsize=(10,10))\n        plt.scatter(x=tsne_result[:,0],\n                    y=tsne_result[:,1],\n                    c=test_label,\n                    cmap=matplotlib.colors.ListedColormap(colors))\n        color_red=mpatches.Patch(color='red',\n                                 label='Insincere Questions')\n        color_orange=mpatches.Patch(color='orange',\n                                    label='Sincere Questions')\n        plt.legend(handles=[color_orange,\n                            color_red])\n        plt.title(\"PCA\")\n        plt.show() \n    if(option==3):\n        pca_result=pca.fit_transform(test_data.toarray())\n        plt.figure(figsize=(10,8))\n        colors=['orange','red']\n        sns.scatterplot(x=pca_result[:,0],\n                        y=pca_result[:,1],\n                        hue=test_label)\n        plt.show()\n        plt.figure(figsize=(10,10))\n        plt.scatter(x=pca_result[:,0],\n                    y=pca_result[:,1],\n                    c=test_label,\n                    cmap=matplotlib.colors.ListedColormap(colors))\n        color_red=mpatches.Patch(color='red',label='Insincere Questions')\n        color_orange=mpatches.Patch(color='orange',\n                                    label='Sincere Questions')\n        plt.legend(handles=[color_orange,color_red])\n        plt.title(\"TSNE\")\n        plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data=train       \n\ndata_vect=train_data['question_text'].values\n\ndata_vect_good=count_good['question_text'].values\n\ntarget_vect=train_data['target'].values\n\ntarget_data_vect_good=train[train['target']==0].values\n\ndata_vect_bad=count_bad['question_text'].values\n\ntarget_data_vect_bad=train[train['target']==1].values\n\ntrain_data_cv,cv= vectorize(data_vect)\n\nreal_review_train_data_cv,cv=vectorize(data_vect_good)\n\nprint(train_data.head())\n\ndimen_reduc_plot(train_data_cv,target_vect,1)\n\ndimen_reduc_plot(real_review_train_data_cv,target_data_vect_good,1)\n\ndimen_reduc_plot(real_review_train_data_cv,target_data_vect_bad,1)\n\n# dimen_reduc_plot(train_data_cv,target_vect,3)\n# dimen_reduc_plot(real_review_train_data_cv,target_data_vect_good,3)\n# dimen_reduc_plot(train_data_cv,target_vect,2)\n# dimen_reduc_plot(real_review_train_data_cv,target_data_vect_good,2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TSNE visualization"},{"metadata":{"trusted":true},"cell_type":"code","source":"#TSNE visualization on first 1000 samples\ntrain_data=train[:1000]       \n\ndata_vect=train_data['question_text'].values\n\ndata_vect_good=count_good['question_text'].values\n\ntarget_vect=train_data['target'].values\n\ntarget_data_vect_good=train[train['target']==1].values\n\ndata_vect_bad=count_bad['question_text'].values\n\ntarget_data_vect_bad=train[train['target']==1].values\n\ntrain_data_cv,cv= vectorize(data_vect)\n\nreal_review_train_data_cv,cv=vectorize(data_vect_good)\n\ndimen_reduc_plot(train_data_cv,target_vect,3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"check_df=list(train['question_text'].str.split())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Semantic Embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n## Load word2vec algorithm from gensim\nfrom gensim.models import Word2Vec,KeyedVectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model=Word2Vec(check_df,min_count=1)\nword_li=list(model.wv.vocab)\nprint(word_li[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#View the Tensor\nprint(model)\nprint(model['questions'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##save the modeled words produced from Word2Vec\nmodel.save('word2vec_model.bin')\nloaded_model=KeyedVectors.load('word2vec_model.bin')\nprint(loaded_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#View the Embedding Word Vector\nplt.plot(model['questions'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Dimension reduction of the embedding vectors"},{"metadata":{"trusted":true},"cell_type":"code","source":"train.sample()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Measure Cosine distance\ndistance=model.similarity('questions','insincere')\nprint(distance)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PCA transform in 2D for visualization of embedded words\nfrom matplotlib import pyplot\n\npca = PCA(n_components=2)\n\ntransformation_model=loaded_model[loaded_model.wv.vocab]\n\nresult = pca.fit_transform(transformation_model[:50])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# create a scatter plot of the projection\n\npyplot.scatter(result[:, 0], result[:, 1])\nwords = list(loaded_model.wv.vocab)\nfor i, word in enumerate(words[:50]):\n    pyplot.annotate(word, xy=(result[i, 0], result[i, 1]))\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Using Google News Embeddings For our corpus\ngoogle_news_embed='../input/nlpword2vecembeddingspretrained/GoogleNews-vectors-negative300.bin'\ngoogle_loaded_model=KeyedVectors.load_word2vec_format(google_news_embed,binary=True)\nprint(google_loaded_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Visualize the Word Vectors\nplt.plot(google_loaded_model['questions'])\nplt.plot(google_loaded_model['insincere'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## PCA transform in 2D for visualization of google news embedded words"},{"metadata":{"trusted":true},"cell_type":"code","source":"from matplotlib import pyplot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pca = PCA(n_components=2)\n\ntransformation_model=google_loaded_model[google_loaded_model.wv.vocab]\n\nresult = pca.fit_transform(transformation_model[:50])\n\n# create a scatter plot of the projection\npyplot.scatter(result[:, 0], result[:, 1])\n\nwords = list(google_loaded_model.wv.vocab)\n\nfor i, word in enumerate(words[:50]):\n    pyplot.annotate(word, xy=(result[i, 0], result[i, 1]))\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"   \n# Working with Glove Embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.scripts.glove2word2vec import glove2word2vec\n\nglove_file='../input/nlpword2vecembeddingspretrained/glove.6B.50d.txt'\nword2vec_output_file = 'glove.6B.100d.txt.word2vec'\nglove_loaded=glove2word2vec(glove_file, word2vec_output_file)\nprint(glove_loaded)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"glove_model = KeyedVectors.load_word2vec_format(word2vec_output_file, binary=False)\nplt.plot(glove_model['questions'])\nplt.plot(glove_model['insincere'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PCA transform in 2D for visualization of glove embedded words\nfrom matplotlib import pyplot\npca = PCA(n_components=2)\n\ntransformation_model=glove_model[glove_model.wv.vocab]\nresult = pca.fit_transform(transformation_model[:50])\n\n# create a scatter plot of the projection\npyplot.scatter(result[:, 0], result[:, 1])\nwords = list(glove_model.wv.vocab)\nfor i, word in enumerate(words[:50]):\n    pyplot.annotate(word, xy=(result[i, 0], result[i, 1]))\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Using Fasttext"},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import Word2Vec,KeyedVectors\n\nfasttext_file=\"../input/fasttext-crawl-300d-2m/crawl-300d-2M.vec\"\nprint(fasttext_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fasttext_model = KeyedVectors.load_word2vec_format(fasttext_file, binary=False)\nplt.plot(fasttext_model['questions'])\nplt.plot(fasttext_model['insincere'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PCA transform in 2D for visualization of glove embedded words\nfrom matplotlib import pyplot\npca = PCA(n_components=2)\n\ntransformation_model=fasttext_model[fasttext_model.wv.vocab]\nresult = pca.fit_transform(transformation_model[:50])\n\n# create a scatter plot of the projection\npyplot.scatter(result[:, 0], result[:, 1])\nwords = list(fasttext_model.wv.vocab)\nfor i, word in enumerate(words[:50]):\n    pyplot.annotate(word, xy=(result[i, 0], result[i, 1]))\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Using Keras"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow import keras\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing.text import Tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Creating Embedding Matrix\nmaxlen=1000\nmax_features=5000\nembed_size=300\n\ntrain_sample=train['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Tokenizing steps- must be remembered\ntokenizer=Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_sample))\ntrain_sample=tokenizer.texts_to_sequences(train_sample)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Pad the sequence- To allow same length for all vectorized words\ntrain_sample=pad_sequences(train_sample,maxlen=maxlen)\n\nEMBEDDING_FILE = '../input/wikinews300d1msubwordvec/wiki-news-300d-1M-subword.vec'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\nplt.plot(embedding_matrix[20])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Compeletion of static embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}