{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport re\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"sample = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/sample_submission.csv')\ntrain = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train),train.index.shape[-1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn import preprocessing,metrics,manifold\nfrom sklearn.manifold import TSNE\nfrom sklearn.model_selection import train_test_split,cross_val_score,GridSearchCV,cross_val_predict\nfrom imblearn.over_sampling import ADASYN,SMOTE\nfrom imblearn.under_sampling import NearMiss\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.svm import SVC\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.discriminant_analysis import LinearDiscriminantAnalysis\nfrom sklearn.svm import SVC\nimport collections\nimport matplotlib.patches as mpatches\nfrom sklearn.metrics import accuracy_score\n%matplotlib inline\nfrom sklearn.preprocessing import RobustScaler\nimport xgboost\nfrom imblearn.metrics import classification_report_imbalanced\nfrom sklearn.metrics import classification_report,roc_auc_score,roc_curve,r2_score,recall_score,confusion_matrix,precision_recall_curve\nfrom collections import Counter\nfrom sklearn.model_selection import StratifiedKFold,KFold,StratifiedShuffleSplit\nfrom nltk import word_tokenize\nfrom nltk.corpus import stopwords\nstop_words = stopwords.words('english')\nimport matplotlib.pyplot as plt\nimport matplotlib.patches as mpatches\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.decomposition import PCA, TruncatedSVD,SparsePCA\nfrom sklearn.metrics import classification_report,confusion_matrix\nfrom nltk.tokenize import word_tokenize\nfrom collections import defaultdict\nfrom collections import Counter\nimport seaborn as sns\nfrom wordcloud import WordCloud,STOPWORDS\nimport nltk\nfrom nltk.corpus import stopwords\nimport string\nfrom plotly import subplots\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"The Shape of the Dataset\".format(),train.shape)\n# Shape of the data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"good_reviews=train[train['target']==0]['question_text']\nbad_reviews=train[train['target']==1]['question_text']\nprint(\"First 10 samples of good question text\\n\".format(),good_reviews[:10])\nprint(\"First 10 samples of bad question text\\n\".format(),bad_reviews[:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Count of good and bad reviews\ncount=train['target'].value_counts()\nprint('Total Counts of both sets'.format(),count)\n\nprint(\"==============\")\n#Creating a function to plot the counts using matplotlib\ndef plot_counts(count_good,count_bad):\n    plt.rcParams['figure.figsize']=(6,6)\n    plt.bar(0,count_good,width=0.6,label='Positive Reviews',color='Green')\n    plt.legend()\n    plt.bar(2,count_bad,width=0.6,label='Negative Reviews',color='Red')\n    plt.legend()\n    plt.ylabel('Count of Reviews')\n    plt.xlabel('Types of Reviews')\n    plt.show()\n    \ncount_good=train[train['target']==0]\ncount_bad=train[train['target']==1]\nplot_counts(len(count_good),len(count_bad))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Analyse the count of words in each segment- both positive and negative reviews\n\n\n#Create generic plotter with Seaborn\ndef plot_count(count_ones,count_zeros,title_1,title_2,subtitle):\n    fig,(ax1,ax2)=plt.subplots(1,2,figsize=(15,5))\n    sns.distplot(count_zeros,ax=ax1,color='Blue')\n    ax1.set_title(title_1)\n    sns.distplot(count_ones,ax=ax2,color='Red')\n    ax2.set_title(title_2)\n    fig.suptitle(subtitle)\n    plt.show()    \n\n\n\n\ncount_good_words=count_good['question_text'].str.split().apply(lambda z:len(z))\ncount_bad_words=count_bad['question_text'].str.split().apply(lambda z:len(z))\nprint(\"Positive Review Words:\" + str(count_good_words))\nprint(\"Negative Review Words:\" + str(count_bad_words))\nplot_count(count_good_words,count_bad_words,\"Positive Review\",\"Negative Review\",\"Reviews Word Analysis\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Count Punctuations/Stopwords/Codes and other semantic datatypes\n#We will be using the \"generic_plotter\" function.\n\ncount_good_punctuations=count_good['question_text'].apply(lambda z: len([c for c in str(z) if c in string.punctuation]))\ncount_bad_punctuations=count_bad['question_text'].apply(lambda z:len([c for c in str(z) if c in string.punctuation]))\nplot_count(count_good_punctuations,count_bad_punctuations,\"Positive Review Punctuations\",\"Negative Review Punctuations\",\"Reviews Word Punctuation Analysis\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_good_punctuations.value_counts(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Analyse Stopwords\n\ndef plot_count_1(count_ones,count_zeros,title_1,title_2,subtitle):\n    fig,(ax1,ax2)=plt.subplots(1,2,figsize=(15,5))\n    sns.distplot(count_zeros,ax=ax1,color='Blue')\n    ax1.set_title(title_1)\n    sns.distplot(count_ones,ax=ax2,color='Orange')\n    ax2.set_title(title_2)\n    fig.suptitle(subtitle)\n    plt.show()    \n\n\nstops=set(stopwords.words('english'))\ncount_good_stops=count_good['question_text'].apply(lambda z : np.mean([len(z) if w in stops else 0 for w in str(z).split()]))\ncount_bad_stops=count_bad['question_text'].apply(lambda z : np.mean([len(z) if w in stops else 0 for w in str(z).split()]))\nplot_count_1(count_good_stops,count_bad_stops,\"Positive Question Stopwords\",\"Negative Question Stopwords\",\"Question Stopwords Analysis\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_bad_stops.value_counts(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#WordCloud Visualizations\n#Method for creating wordclouds\n\ndef display_cloud(data,color,figsize):\n    plt.subplots(figsize=figsize)\n    wc = WordCloud(stopwords=STOPWORDS,background_color=\"white\", contour_width=2, contour_color=color,\n                   max_words=2000, max_font_size=256,\n                   random_state=42)\n    wc.generate(' '.join(data))\n    plt.imshow(wc, interpolation=\"bilinear\")\n    plt.axis('off')\n    plt.show()\n    \ndisplay_cloud(train['question_text'],color='red',figsize=(15,15))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_cloud(count_good['question_text'],color='red',figsize=(15,15))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_cloud(count_bad['question_text'],'blue',figsize=(15,15))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Simplified counter function\ndef create_corpus(word):\n    corpus=[]\n    \n    for x in train[train['target']==word]['question_text'].str.split():\n        for i in x:\n            corpus.append(i)\n    return corpus\n\ncorpus=create_corpus(1)\ncounter=Counter(corpus)\nmost=counter.most_common()\nx=[]\ny=[]\nfor word,count in most[:100]:\n    if (word not in stops) :\n        x.append(word)\n        y.append(count)\nsns.barplot(x=y,y=x)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Inference From Analysis -I\nThe following can be inferred from the data:\n\nThe dataset is unbalanced.\nThe dataset contains unequal number of semantics for reviews of both polarity.\nThe dataset doesn't contains redundant words and html syntaxes.\nPunctuations/stopwords are present in a equal distribution in the dataset.\nThis tells us that we have to do lots of cleaning!"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Gram analysis on Training set- Bigram and Trigram\nstopword=set(stopwords.words('english'))\ndef gram_analysis(data,gram):\n    tokens=[t for t in data.lower().split(\" \") if t!=\"\" if t not in stopword]\n    ngrams=zip(*[tokens[i:] for i in range(gram)])\n    final_tokens=[\" \".join(z) for z in ngrams]\n    return final_tokens\n\n\n#Create frequency grams for analysis\n    \ndef create_dict(data,grams):\n    freq_dict=defaultdict(int)\n    for sentence in data:\n        for tokens in gram_analysis(sentence,grams):\n            freq_dict[tokens]+=1\n    return freq_dict\n\ndef horizontal_bar_chart(df, color):\n    trace = go.Bar(\n        y=df[\"n_gram_words\"].values[::-1],\n        x=df[\"n_gram_frequency\"].values[::-1],\n        showlegend=False,\n        orientation = 'h',\n        marker=dict(\n            color=color,\n        ),\n    )\n    return trace\n\n\n\ndef create_new_df(freq_dict,):\n    freq_df=pd.DataFrame(sorted(freq_dict.items(),key=lambda z:z[1])[::-1])\n    freq_df.columns=['n_gram_words','n_gram_frequency']\n    #print(freq_df.head())\n    #plt.barh(freq_df['n_gram_words'][:20],freq_df['n_gram_frequency'][:20],linewidth=0.3)\n    #plt.show()\n    trace=horizontal_bar_chart(freq_df[:20],'orange')\n    return trace\n    \ndef plot_grams(trace_zero,trace_one):\n    fig = subplots.make_subplots(rows=1, cols=2, vertical_spacing=0.04,\n                          subplot_titles=[\"Frequent words of positive Questions\", \n                                          \"Frequent words of negative Questions\"])\n    fig.append_trace(trace_zero, 1, 1)\n    fig.append_trace(trace_ones, 1, 2)\n    fig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title=\"Word Count Plots\")\n    py.iplot(fig, filename='word-plots')\n    \n    \ntrain_df_zero=count_bad['question_text']\ntrain_df_ones=count_good['question_text']\n\nprint(\"Bi-gram analysis\")\nfreq_train_df_zero=create_dict(train_df_zero[:200],2)\n#print(freq_train_df_zero)\ntrace_zero=create_new_df(freq_train_df_zero)\nfreq_train_df_ones=create_dict(train_df_ones[:200],2)\n#print(freq_train_df_zero)\ntrace_ones=create_new_df(freq_train_df_ones)\nplot_grams(trace_zero,trace_ones)\nprint(\"Tri-gram analysis\")\nfreq_train_df_zero=create_dict(train_df_zero[:200],3)\n#print(freq_train_df_zero)\ntrace_zero=create_new_df(freq_train_df_zero)\nfreq_train_df_ones=create_dict(train_df_ones[:200],3)\n#print(freq_train_df_zero)\ntrace_ones=create_new_df(freq_train_df_ones)\nplot_grams(trace_zero,trace_ones)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Inference from Analysis - II\nIn this section, we have analysed based on positional features of words in a corpus/sentence/paragraph. The Gram analysis,particularly the pentagram analysis provides an idea which sentences occur more often in the corpus. And in most of the cases, these bag of words are the ones picked up by any frequency vectorization technique.\n\nThus this provides an outline as to the frequency of the conjuction of words which are occuring at the highest frequency. Another important aspect is that,there is a presence of certain html tags and punctuations which have to be removed as these are adding noise to the review corpus. This will be taken up in the cleaning phase.\n\nTime for some Cleaning!\nBefore we move ahead , let us clean the dataset and remove the redundancies.This includes\n\n\n- Emojis\n- Stopwords\n- Punctuations\n- Expanding Abbreviations\n\nThese will be sufficient for cleaning the corpus!\n\nRegex is a very good tool which will help us to do this cleaning."},{"metadata":{"trusted":true},"cell_type":"code","source":"class TextCleaningUtils:\n    '''\n        This class contains implementations of various text cleaning operations (Static Methods)\n    '''\n\n    cleaning_regex_map = {\n        'web_links': r'(?i)(?:(?:http(?:s)?:)|(?:www\\.))\\S+',\n#         \n        'special_chars': r'[^a-zA-Z\\s\\.,!?;:]+',  ## removing nums\n        'redundant_spaces': r'\\s\\s+',\n        'redundant_newlines': r'[\\r|\\n|\\r\\n]+',\n        'twitter_handles': r'[#@]\\S+',\n        'punctuations': r'[\\.,!?;:]+'\n    }\n    \n    def expand_abbreviations(text):\n        text = re.sub(r\"he's\", \"he is\", text)\n        text = re.sub(r\"there's\", \"there is\", text)\n        text = re.sub(r\"We're\", \"We are\", text)\n        text = re.sub(r\"That's\", \"That is\", text)\n        text = re.sub(r\"won't\", \"will not\", text)\n        text = re.sub(r\"they're\", \"they are\", text)\n        text = re.sub(r\"Can't\", \"Cannot\", text)\n        text = re.sub(r\"wasn't\", \"was not\", text)\n        text = re.sub(r\"don\\x89Ûªt\", \"do not\", text)\n        text= re.sub(r\"aren't\", \"are not\", text)\n        text = re.sub(r\"isn't\", \"is not\", text)\n        text = re.sub(r\"What's\", \"What is\", text)\n        text = re.sub(r\"haven't\", \"have not\", text)\n        text = re.sub(r\"hasn't\", \"has not\", text)\n        text = re.sub(r\"There's\", \"There is\", text)\n        text = re.sub(r\"He's\", \"He is\", text)\n        text = re.sub(r\"It's\", \"It is\", text)\n        text = re.sub(r\"You're\", \"You are\", text)\n        text = re.sub(r\"I'M\", \"I am\", text)\n        text = re.sub(r\"shouldn't\", \"should not\", text)\n        text = re.sub(r\"wouldn't\", \"would not\", text)\n        text = re.sub(r\"couldn't\", \"could not\", text)\n        text = re.sub(r\"i'm\", \"I am\", text)\n        text = re.sub(r\"I\\x89Ûªm\", \"I am\", text)\n        text = re.sub(r\"I'm\", \"I am\", text)\n        text = re.sub(r\"Isn't\", \"is not\", text)\n        text = re.sub(r\"Here's\", \"Here is\", text)\n        text = re.sub(r\"you've\", \"you have\", text)\n        text = re.sub(r\"you\\x89Ûªve\", \"you have\", text)\n        text = re.sub(r\"we're\", \"we are\", text)\n        text = re.sub(r\"what's\", \"what is\", text)\n        text = re.sub(r\"couldn't\", \"could not\", text)\n        text = re.sub(r\"we've\", \"we have\", text)\n        text = re.sub(r\"it\\x89Ûªs\", \"it is\", text)\n        text = re.sub(r\"doesn\\x89Ûªt\", \"does not\", text)\n        text = re.sub(r\"It\\x89Ûªs\", \"It is\", text)\n        text = re.sub(r\"Here\\x89Ûªs\", \"Here is\", text)\n        text = re.sub(r\"who's\", \"who is\", text)\n        text = re.sub(r\"I\\x89Ûªve\", \"I have\", text)\n        text = re.sub(r\"y'all\", \"you all\", text)\n        text = re.sub(r\"can\\x89Ûªt\", \"cannot\", text)\n        text = re.sub(r\"would've\", \"would have\", text)\n        text = re.sub(r\"it'll\", \"it will\", text)\n        text = re.sub(r\"we'll\", \"we will\", text)\n        text = re.sub(r\"wouldn\\x89Ûªt\", \"would not\", text)\n        text = re.sub(r\"We've\", \"We have\", text)\n        text = re.sub(r\"he'll\", \"he will\", text)\n        text = re.sub(r\"Y'all\", \"You all\", text)\n        text = re.sub(r\"Weren't\", \"Were not\", text)\n        text = re.sub(r\"Didn't\", \"Did not\", text)\n        text = re.sub(r\"they'll\", \"they will\", text)\n        text = re.sub(r\"DON'T\", \"DO NOT\", text)\n        text = re.sub(r\"That\\x89Ûªs\", \"That is\", text)\n        text = re.sub(r\"they've\", \"they have\", text)\n        text = re.sub(r\"they'd\", \"they would\", text)\n        text = re.sub(r\"i'd\", \"I would\", text)\n        text = re.sub(r\"should've\", \"should have\", text)\n        text = re.sub(r\"You\\x89Ûªre\", \"You are\", text)\n        text = re.sub(r\"where's\", \"where is\", text)\n        text = re.sub(r\"Don\\x89Ûªt\", \"Do not\", text)\n        text = re.sub(r\"i'll\", \"I will\", text)\n        text = re.sub(r\"weren't\", \"were not\", text)\n        text = re.sub(r\"They're\", \"They are\", text)\n        text = re.sub(r\"Can\\x89Ûªt\", \"Cannot\", text)\n        text = re.sub(r\"you\\x89Ûªll\", \"you will\", text)\n        text = re.sub(r\"I\\x89Ûªd\", \"I would\", text)\n        text = re.sub(r\"let's\", \"let us\", text)\n        text = re.sub(r\"it's\", \"it is\", text)\n        text = re.sub(r\"can't\", \"cannot\", text)\n        text = re.sub(r\"don't\", \"do not\", text)\n        text = re.sub(r\"you're\", \"you are\", text)\n        text = re.sub(r\"i've\", \"I have\", text)\n        text = re.sub(r\"that's\", \"that is\", text)\n        text = re.sub(r\"i'll\", \"I will\", text)\n        text = re.sub(r\"doesn't\", \"does not\",text)\n        text = re.sub(r\"i'd\", \"I would\", text)\n        text = re.sub(r\"didn't\", \"did not\", text)\n        text = re.sub(r\"ain't\", \"am not\", text)\n        text = re.sub(r\"you'll\", \"you will\", text)\n        text = re.sub(r\"I've\", \"I have\", text)\n        text = re.sub(r\"Don't\", \"do not\", text)\n        text = re.sub(r\"I'll\", \"I will\", text)\n        text = re.sub(r\"I'd\", \"I would\", text)\n        text = re.sub(r\"Let's\", \"Let us\", text)\n        text = re.sub(r\"you'd\", \"You would\", text)\n        text = re.sub(r\"It's\", \"It is\", text)\n        text = re.sub(r\"Ain't\", \"am not\", text)\n        text = re.sub(r\"Haven't\", \"Have not\", text)\n        text = re.sub(r\"Hadn't\", \"Had not\", text)\n        text = re.sub(r\"Could've\", \"Could have\", text)\n        text = re.sub(r\"youve\", \"you have\", text)  \n        text = re.sub(r\"donå«t\", \"do not\", text)  \n\n        return text\n    \n    def remove_emojis(text):\n        emoji_clean= re.compile(\"[\"\n                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                           u\"\\U00002702-\\U000027B0\"\n                           u\"\\U000024C2-\\U0001F251\"\n                           \"]+\", flags=re.UNICODE)\n        text=emoji_clean.sub(r'',text)\n        url_clean= re.compile(r\"https://\\S+|www\\.\\S+\")\n        text=url_clean.sub(r'',text)\n        return text\n    \n\n\n    @staticmethod\n    def clean_text_from_regex(text, text_clean_regex):\n        '''\n            Follow a particular cleaning expression, provided\n            as an input by an user to clean the text.\n        '''\n\n        text = text_clean_regex.sub(' ', text).strip()\n        return text\n\n    @staticmethod\n    def strip_html(text):\n        soup = BeautifulSoup(text, \"html.parser\")\n        return soup.get_text()\n\n    @staticmethod\n    def remove_special_chars(text):\n        '''\n            Replace any special character provided as default,\n            which is present in the text with space\n        '''\n\n        special_chars_regex = re.compile(TextCleaningUtils.cleaning_regex_map['special_chars'])\n        text = TextCleaningUtils.clean_text_from_regex(text, special_chars_regex)\n        return text\n\n    @staticmethod\n    def remove_redundant_spaces(text):\n        '''\n            Remove any redundant space provided as default,\n            that is present in the text.\n        '''\n\n        redundant_spaces_regex = re.compile(\n            TextCleaningUtils.cleaning_regex_map['redundant_spaces'])\n        text = TextCleaningUtils.clean_text_from_regex(text, redundant_spaces_regex)\n        return text\n\n    @staticmethod\n    def remove_web_links(text):\n        '''\n            Removes any web link that follows a particular default expression,\n            present in the text.\n        '''\n\n        web_links_regex = re.compile(TextCleaningUtils.cleaning_regex_map['web_links'])\n        text = TextCleaningUtils.clean_text_from_regex(text, web_links_regex)\n        return text\n\n    @staticmethod\n    def remove_twitter_handles(text):\n        '''\n            Removes any twitter handle present in the text.\n        '''\n\n        twitter_handles_regex = re.compile(TextCleaningUtils.cleaning_regex_map['twitter_handles'])\n        text = TextCleaningUtils.clean_text_from_regex(text, twitter_handles_regex)\n        return text\n\n    @staticmethod\n    def remove_redundant_newlines(text):\n        '''\n            Removes any redundant new line present in the text.\n        '''\n\n        redundant_newlines_regex = re.compile(\n            TextCleaningUtils.cleaning_regex_map['redundant_newlines'])\n        text = TextCleaningUtils.clean_text_from_regex(text, redundant_newlines_regex)\n        return text\n\n    @staticmethod\n    def remove_punctuations(text):\n        '''\n            Removes any punctuation that follows the default expression, in the text.\n        '''\n\n        remove_punctuations_regex = re.compile(TextCleaningUtils.cleaning_regex_map['punctuations'])\n        text = TextCleaningUtils.clean_text_from_regex(text, remove_punctuations_regex)\n        return text\n\n    @staticmethod\n    def remove_exaggerated_words(text):\n        '''\n            Removes any exaggerated word present in the text.\n        '''\n\n        return ''.join(''.join(s)[:2] for _, s in itertools.groupby(text))\n\n    @staticmethod\n    def replace_multiple_chars(text):\n        '''\n            Replaces multiple characters present in the text.\n        '''\n\n        char_list = ['.', '?', '!', '#', '$', '/', '@', '*', '(', ')', '+']\n        final_text = ''\n        for i in char_list:\n            if i in text:\n                pattern = \"\\\\\" + i + '{2,}'\n                repl_str = i.replace(\"\\\\\", \"\")\n                text = re.sub(pattern, repl_str, text)\n                final_text = ' '.join(text.split())\n        return final_text\n\n    @staticmethod\n    def replace_sign(text):\n        '''\n            Replaces any sign with words like & with 'and', in the text.\n        '''\n        sign_list = {'&': ' and ', '/': ' or ', '\\xa0': ' '}\n        final_text = ''\n        for i in sign_list:\n            if i in text:\n                text = re.sub(i, sign_list[i], text)\n                final_text = ' '.join(text.split())\n        return final_text\n\n    @staticmethod\n    def remove_accented_char(text):\n        text = unicodedata.normalize('NFD', text) \\\n            .encode('ascii', 'ignore') \\\n            .decode(\"utf-8\")\n        return str(text)\n\n    @staticmethod\n    def replace_characters(text, replace_map):\n        '''\n            Replaces any character custom provided by an user.\n        '''\n\n        for char, replace_val in replace_map.items():\n            text = text.replace(char, replace_val)\n        return text\n    \ndef clean_data(df,col_to_clean):\n    df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.expand_abbreviations)\n    df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_emojis)\n    df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_special_chars)\n    df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_redundant_spaces)\n    df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_punctuations)\n    df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_exaggerated_words)\n    df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_redundant_newlines)\n    df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_twitter_handles)\n    df[col_to_clean] = df[col_to_clean].apply(TextCleaningUtils.remove_web_links)\n    df[col_to_clean] = df[col_to_clean].astype(str)\n    df[col_to_clean] = df[col_to_clean].str.lower()\n    \n    return (df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import itertools\nfrom bs4 import BeautifulSoup","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_good_df = clean_data(count_good, 'question_text')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_bad_df = clean_data(count_bad, 'question_text')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = clean_data(train, 'question_text')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['target'].value_counts(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"count_bad","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Apply Gram Analysis\ntrain_df_zero=count_bad_df['question_text']\ntrain_df_ones=count_good_df['question_text']\nprint(\"Tri-gram analysis\")\nfreq_train_df_zero=create_dict(train_df_zero[:200],3)\n#print(freq_train_df_zero)\ntrace_zero=create_new_df(freq_train_df_zero)\nfreq_train_df_ones=create_dict(train_df_ones[:200],3)\n#print(freq_train_df_zero)\ntrace_ones=create_new_df(freq_train_df_ones)\nplot_grams(trace_zero,trace_ones)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(\"Penta-gram analysis\")\nfreq_train_df_zero=create_dict(train_df_zero[:200],5)\n#print(freq_train_df_zero)\ntrace_zero=create_new_df(freq_train_df_zero)\nfreq_train_df_ones=create_dict(train_df_ones[:200],5)\n#print(freq_train_df_zero)\ntrace_ones=create_new_df(freq_train_df_ones)\nplot_grams(trace_zero,trace_ones)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_cloud(train_df['question_text'],color='red',figsize=(15,15))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_cloud(count_bad_df['question_text'],color='red',figsize=(15,15))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"display_cloud(count_good_df['question_text'],color='red',figsize=(15,15))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Lemmatize the dataset\nfrom nltk.stem import WordNetLemmatizer\n\n\ndef lemma_traincorpus(data):\n    lemmatizer=WordNetLemmatizer()\n    out_data=\"\"\n    for words in data:\n        out_data+= lemmatizer.lemmatize(words)\n    return out_data\n\ntrain_df['question_text'] = train_df['question_text'].apply(lambda z: lemma_traincorpus(z))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#For example let us try to stem them and check  a sample\n\nfrom nltk.stem import *\ndef stem_traincorpus(data):\n    stemmer = PorterStemmer()\n    out_data=\"\"\n    for words in data:\n        out_data+= stemmer.stem(words)\n    return out_data\n\nsample_train_df=train_df[5:10]\nsample_train_df['question_text']=sample_train_df['question_text'].apply(lambda z: stem_traincorpus(z))\nsample_train_df['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntfidf_vect=TfidfVectorizer(stop_words='english',ngram_range=(1,3))\ntrain_tfidf=tfidf_vect.fit_transform(train_df['question_text'].values.tolist())\ntrain_tfidf.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Outputs from the TF-IDF transformed data\nprint(train_tfidf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Count Vectorization\nimport matplotlib\nimport matplotlib.pyplot as plt\ndef vectorize(data):\n    cv=CountVectorizer()\n    fit_data_cv=cv.fit_transform(data)\n    return fit_data_cv,cv\n\n#Tfidf vectorization from sklearn\ndef tfidf(data):\n    tfidfv=TfidfVectorizer()\n    fit_data_tfidf=tfidfv.fit_transform(data)\n    return fit_data_cv,tfidfv\n\ndef dimen_reduc_plot(test_data,test_label,option):\n    tsvd= TruncatedSVD(n_components=2,algorithm=\"randomized\",random_state=42)\n    tsne=TSNE(n_components=2,random_state=42) #not recommended instead use PCA\n    pca=SparsePCA(n_components=2,random_state=42)\n    if(option==1):\n        tsvd_result=tsvd.fit_transform(test_data)\n        plt.figure(figsize=(10,8))\n        colors=['orange','red']\n        \n        sns.scatterplot(x=tsvd_result[:,0],y=tsvd_result[:,1],hue=test_label        )\n        \n        plt.show()\n        plt.figure(figsize=(10,10))\n        plt.scatter(tsvd_result[:,0],tsvd_result[:,1],c=test_label,cmap=matplotlib.colors.ListedColormap(colors))\n        color_red=mpatches.Patch(color='red',label='Insincere Question')\n        color_orange=mpatches.Patch(color='orange',label='Sincere Question')\n        plt.legend(handles=[color_orange,color_red])\n        plt.title(\"TSVD\")\n        plt.show()\n    if(option==2):\n        tsne_result=tsne.fit_transform(test_data)\n        plt.figure(figsize=(10,8))\n        colors=['orange','red']\n        sns.scatterplot(x=tsne_result[:,0],y=tsne_result[:,1],hue=test_label)\n        plt.show()\n        plt.figure(figsize=(10,10))\n        plt.scatter(x=tsne_result[:,0],y=tsne_result[:,1],c=test_label,cmap=matplotlib.colors.ListedColormap(colors))\n        color_red=mpatches.Patch(color='red',label='Insincere Question')\n        color_orange=mpatches.Patch(color='orange',label='Sincere Question')\n        plt.legend(handles=[color_orange,color_red])\n        plt.title(\"PCA\")\n        plt.show() \n    if(option==3):\n        pca_result=pca.fit_transform(test_data.toarray())\n        plt.figure(figsize=(10,8))\n        colors=['orange','red']\n        sns.scatterplot(x=pca_result[:,0],y=pca_result[:,1],hue=test_label)\n        plt.show()\n        plt.figure(figsize=(10,10))\n        plt.scatter(x=pca_result[:,0],y=pca_result[:,1],c=test_label,cmap=matplotlib.colors.ListedColormap(colors))\n        color_red=mpatches.Patch(color='red',label='Insincere Question')\n        color_orange=mpatches.Patch(color='orange',label='Sincere Question')\n        plt.legend(handles=[color_orange,color_red])\n        plt.title(\"TSNE\")\n        plt.show()\ntrain_data=train_df       \ndata_vect=train_data['question_text'].values\ndata_vect_good=count_good['question_text'].values\ntarget_vect=train_data['target'].values\ntarget_data_vect_good=train_df[train_df['target']==0]['target'].values\ndata_vect_bad=count_bad['question_text'].values\ntarget_data_vect_bad=train_df[train_df['target']==1]['target'].values\ntrain_data_cv,cv= vectorize(data_vect)\nreal_review_train_data_cv,cv=vectorize(data_vect_good)\n\nprint(train_data.head())\ndimen_reduc_plot(train_data_cv,target_vect,1)\ndimen_reduc_plot(real_review_train_data_cv,target_data_vect_good,1)\n#dimen_reduc_plot(real_review_train_data_cv,target_data_vect_bad,1)\n# dimen_reduc_plot(train_data_cv,target_vect,3)\n# dimen_reduc_plot(real_review_train_data_cv,target_data_vect_good,3)\n# dimen_reduc_plot(train_data_cv,target_vect,2)\n# dimen_reduc_plot(real_review_train_data_cv,target_data_vect_good,2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"check_df=list(train_df['question_text'].str.split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n## Load word2vec algorithm from gensim\nfrom gensim.models import Word2Vec,KeyedVectors\n\nmodel=Word2Vec(check_df,min_count=1)\nword_li=list(model.wv.vocab)\nprint(word_li[:5])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#View the Tensor\nprint(model)\nprint(model['questions'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#View the Embedding Word Vector\nplt.plot(model['questions'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##save the modeled words produced from Word2Vec\nmodel.save('word2vec_model.bin')\nloaded_model=KeyedVectors.load('word2vec_model.bin')\nprint(loaded_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Measure Cosine distance\ndistance=model.similarity('questions','insincere')\nprint(distance)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PCA transform in 2D for visualization of embedded words\nfrom matplotlib import pyplot\npca = PCA(n_components=2)\ntransformation_model=loaded_model[loaded_model.wv.vocab]\nresult = pca.fit_transform(transformation_model[:50])\n# create a scatter plot of the projection\npyplot.scatter(result[:, 0], result[:, 1])\nwords = list(loaded_model.wv.vocab)\nfor i, word in enumerate(words[:50]):\n    pyplot.annotate(word, xy=(result[i, 0], result[i, 1]))\npyplot.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Using Google News Embeddings for the corpus\nfrom gensim.models import Word2Vec\n\n#download Google News Embeddings\ngoogle_news_embed=\"https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz\"\n\ngoogle_loaded_model=KeyedVectors.load_word2vec_format(google_news_embed,binary=True)\nprint(google_loaded_model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.bar([i for i in range(0,300)], [i for i in google_loaded_model['questions']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Visualize the Word Vectors\n##Comparing the similarities of words graphically\nplt.figure(figsize=(20,9))\nplt.bar([i for i in range(0,300)], [i for i in google_loaded_model['questions']])\nplt.bar([i for i in range(0,300)], [i for i in google_loaded_model['insincere']])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# PCA transform in 2D for visualization of google news embedded words\nfrom matplotlib import pyplot\n\nplt.figure(figsize=(20,12))\npca = PCA(n_components=2)\ntransformation_model = google_loaded_model[google_loaded_model.vocab]\nresult = pca.fit_transform(transformation_model[:100])\n# create a scatter plot of the projection\nplt.scatter(result[:, 0], result[:, 1])\nwords = list(loaded_model.wv.vocab)\nfor i, word in enumerate(words[:100]):\n    plt.annotate(word, xy=(result[i, 0], result[i, 1]))\n    plt.title(\"PCA Transformation of word vectors with Google News Embeddings on to 2D-space\\n\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Glove Embeddings\n\n* Glove embeddings rely on global vector representations mechanism, which is an unsupervised algorithm.\n\n* This captures both the global corpus statistics as well as local semantic information.\n\n* Glove vectors used here can be converted from \"txt\" format to Word2Vec format by using scripts provided in the Gensim library.\n\n* This allows us to manipulate the glove embeddings in a manner similar to Word2Vec and apply the similarity metric.\n\n* The loss function for the glove relies on logistic regression of the log co-occurence probabilities."},{"metadata":{"trusted":true},"cell_type":"code","source":"\n'''\nimport zipfile\n\npath_to_zip_file = '../input/quora-insincere-questions-classification/embeddings.zip'\nwith zipfile.ZipFile(path_to_zip_file, 'r') as zip_ref:\n    zip_ref.extractall('./')'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''#Using Google News Embeddings For our corpus\ngoogle_news_embed = './GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\ngoogle_loaded_model=KeyedVectors.load_word2vec_format(google_news_embed,binary=True)\nprint(google_loaded_model)'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''#Visualize the Word Vectors\nplt.plot(google_loaded_model['questions'])\nplt.plot(google_loaded_model['insincere'])\nplt.show()'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''# PCA transform in 2D for visualization of google news embedded words\nfrom matplotlib import pyplot\npca = PCA(n_components=2)\ntransformation_model=google_loaded_model[google_loaded_model.wv.vocab]\nresult = pca.fit_transform(transformation_model[:50])\n# create a scatter plot of the projection\npyplot.scatter(result[:, 0], result[:, 1])\nwords = list(google_loaded_model.wv.vocab)\nfor i, word in enumerate(words[:50]):\n    pyplot.annotate(word, xy=(result[i, 0], result[i, 1]))\npyplot.show()'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#%env JOBLIB_TEMP_FOLDER=/tmp\n\nfrom gensim.scripts.glove2word2vec import glove2word2vec\n\nglove_file='../input/glove6b50dtxt/glove.6B.50d.txt'\nword2vec_output_file = 'glove.6B.50d.txt'\nglove_loaded=glove2word2vec(glove_file, word2vec_output_file)\nprint(glove_loaded)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Comparing the similarities of words graphically\nplt.figure(figsize=(20,8))\nglove_model = KeyedVectors.load_word2vec_format(word2vec_output_file, binary=False)\nplt.bar([i for i in range(0,50)], [i for i in glove_model['questions']])\nplt.bar([i for i in range(0,50)], [i for i in glove_model['insincere']])\nplt.title(\"Visualize the Word Vectors with GloVe Embeddings\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#PCA transform in 2D for visualization of glove embedded words\n\nplt.figure(figsize=(20,15))\npca = PCA(n_components=2)\ntransformation_model = glove_model[glove_model.vocab]\nresult = pca.fit_transform(transformation_model[:100])\n# create a scatter plot of the projection\nplt.scatter(result[:, 0], result[:, 1])\nwords = list(loaded_model.wv.vocab)\nfor i, word in enumerate(words[:100]):\n    plt.annotate(word, xy=(result[i, 0], result[i, 1]))\n    plt.title(\"PCA Transformation of word vectors with GloVe Embeddings on to 2D-space\\n\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## **Fastext embeddings**"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Using the fasttext word embeddding from crawl\n\nfrom gensim.models import Word2Vec,KeyedVectors\n\nfasttext_file=\"../input/fasttext-crawl-300d-2m/crawl-300d-2M.vec\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(20,8))\n\nfasttext_model = KeyedVectors.load_word2vec_format(fasttext_file, binary=False)\n\ny1 = [i for i in fasttext_model['questions']]\ny2 = [i for i in fasttext_model['insincere']]\n\nplt.bar([i for i in range(0,len(y1))], y1)\nplt.bar([i for i in range(0,len(y2))], y2, alpha = 0.3)\nplt.title(\"Visualize the Word Vectors with Fast Text Embeddings\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#PCA transform in 2D for visualization of glove embedded words\n\npca = PCA(n_components=2)\ntransformation_model=fasttext_model[fasttext_model.vocab]\nresult = pca.fit_transform(transformation_model[:50])\n\nplt.figure(figsize=(20,12))\n#create a scatter plot of the projection\npyplot.scatter(result[:, 0], result[:, 1])\nwords = list(fasttext_model.vocab)\nfor i, word in enumerate(words[:50]):\n    plt.annotate(word, xy=(result[i, 0], result[i, 1]))\n    plt.title('PCA Transformation of word vectors with Fast Text Embeddings on to 2D-space\\n')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Procedure to create Embedding Matrix\n\nThe importance of building a Embedding matrix is to have mutual co-occurence embedding probabilities (vectors) of all the words present in the corpus. This are the formats wherein neural network frameworks like Keras,Tensorflow are likely to be used.\n\n* These embedding vectors are then passed through deep learning layers.\n\n* The flexibility of these static embeddings is that they are able to provide a good benchmark on the initial task (classification) in our case.\n\n* These also provide a very good approximation to the amount of percentage accuracy or loss which can be achieved with a coupled Encoder-Decoder/Transformer like architectures.\n\nWe will be using the Keras-Tensorflow framework to build the embedding matrix."},{"metadata":{"trusted":true},"cell_type":"code","source":"#copy of the data with the first column dropped\n# new_train_data = training_data.drop(columns='qid')[:10000].copy()\n\nsample_train_df = train_df.set_index('qid').sample(10000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n#Creating Embedding Matrix\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.preprocessing.text import Tokenizer\n\nmax_len=500\nmax_features=5000 \nembed_size=300\n\ntrain_sample = sample_train_df['question_text']\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Tokenizing steps\ntokenizer=Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_sample))\ntrain_sample=tokenizer.texts_to_sequences(train_sample)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Pad the sequence- To allow same length for all vectorized words\ntrain_sample=pad_sequences(train_sample,maxlen=max_len)\n\n\nEMBEDDING_FILE = '../input/wikinews300d1msubwordvec/wiki-news-300d-1M-subword.vec'\n\ndef get_coefs(word,*arr): \n    return word, np.asarray(arr, dtype='float32')\n\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}