{"cells":[{"metadata":{"trusted":true,"_uuid":"363a47918653c3222ba6822e3e414610709f1106"},"cell_type":"code","source":"'''Contents\n    Basic feature extraction using text data\n        Number of words\n        Number of characters\n        Average word length\n        Number of stopwords\n        Number of special characters\n        Number of numerics\n        Number of uppercase words\n    Basic Text Pre-processing of text data\n        Lower casing\n        Punctuation removal\n        Stopwords removal\n        Frequent words removal\n        Rare words removal\n        Spelling correction\n    Tokenization\n        Stemming\n        Lemmatization\n    Advance Text Processing\n        N-grams\n        Term Frequency\n        Inverse Document Frequency\n        Term Frequency-Inverse Document Frequency (TF-IDF)\n        Bag of Words\n        Hashing with HashingVectorizer\n'''","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nfrom nltk.corpus import stopwords\nfrom textblob import TextBlob\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"174006c5bd7ebea2e23e75fbdaae0902b757ec25"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \", train_df.shape)\nprint(\"Test shape : \", test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"00a0a3f63b8745eee38021d76ab69a3ffad245ba"},"cell_type":"code","source":"train_df.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edef87e5b3e0aea5d696c58a3f769eda0b4c130d"},"cell_type":"code","source":"#Number of Words\n\ntrain_df['word_count'] = train_df['question_text'].apply(lambda x: len(str(x).split(\" \")))\ntrain_df[['question_text','word_count']].head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f26ef536d6a0f4e9fb6676629de26cd2f2e1bb52"},"cell_type":"code","source":"#Number of characters\n\ntrain_df['char_count'] = train_df['question_text'].str.len() ## this also includes spaces\ntrain_df[['question_text','char_count']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78dc4a6d0d9e5640a2ee0e25a48ef1de44312f7a"},"cell_type":"code","source":"#Average Word Length\n\ndef avg_word(sentence):\n  words = sentence.split()\n  return (sum(len(word) for word in words)/len(words))\n\ntrain_df['avg_word'] = train_df['question_text'].apply(lambda x: avg_word(x))\ntrain_df[['question_text','avg_word']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"263a2d392fcfeded3d8c038253b1bbf2d808a2bc"},"cell_type":"code","source":"#Number of stopwords\n\nstop = stopwords.words('english')\ntrain_df['stopwords'] = train_df['question_text'].apply(lambda x: len([x for x in x.split() if x in stop]))\ntrain_df[['question_text','stopwords']].head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79a8cc7eeb8f59afc62751be98a800b616a956a7"},"cell_type":"code","source":"#Number of special characters\n\ntrain_df['hastags'] = train_df['question_text'].apply(lambda x: len([x for x in x.split() if x.startswith('#')]))\ntrain_df[['question_text','hastags']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11dadd7263851645a9cc5851023dad9af308db91"},"cell_type":"code","source":"#Number of numerics\n\ntrain_df['numerics'] = train_df['question_text'].apply(lambda x: len([x for x in x.split() if x.isdigit()]))\ntrain_df[['question_text','numerics']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a56eba6cecf695265680b63715d9fefb29adaec"},"cell_type":"code","source":"# Number of Uppercase words\n\ntrain_df['upper'] = train_df['question_text'].apply(lambda x: len([x for x in x.split() if x.isupper()]))\ntrain_df[['question_text','upper']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0dca1a8e1f4b7949a9d6ddeb304433bcc3b0e796"},"cell_type":"code","source":"#Lower case\n\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: x.lower())\ntest_df['question_text'] = test_df['question_text'].apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7bfd9a2a2c0af175cbbcec97333ca8797c7649c5"},"cell_type":"code","source":"#Removing Punctuation\n\ntrain_df['question_text'] = train_df['question_text'].str.replace('[^\\w\\s]','')\ntrain_df['question_text'].head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4c5a160c794c62e2870210ead2522606ae607198"},"cell_type":"code","source":"#Removal of Stop Words\n\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))\ntrain_df['question_text'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97de2bd75fa9fe09c271c20afa0c2668e5cb47b8"},"cell_type":"code","source":"#Common word removal\n\nfreq = pd.Series(' '.join(train_df['question_text']).split()).value_counts()[:10]\nfreq = list(freq.index)\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: \" \".join(x for x in x.split() if x not in freq))\ntrain_df['question_text'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e4838adcbe40d23d1f3f57fbbacf46eb183e139"},"cell_type":"code","source":"#Rare words removal\n\nfreq = pd.Series(' '.join(train_df['question_text']).split()).value_counts()[-10:]\nfreq = list(freq.index)\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: \" \".join(x for x in x.split() if x not in freq))\ntrain_df['question_text'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a8105bc0a7753aead5c450f673d30292085bb8c3"},"cell_type":"code","source":"#Spelling correction\n# train_df['question_text'] = train_df['question_text'].apply(lambda x: str(TextBlob(x).correct()))\n# train_df['question_text'].head()\n\ntrain_df['question_text'][:10].apply(lambda x: str(TextBlob(x).correct()))\ntrain_df['question_text'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"39c8336cd2d2924aa34c57e446c129aeae92d3c6"},"cell_type":"code","source":"#Tokenization\n'''Tokenization refers to dividing the text into a sequence of words or sentences.'''\n\nTextBlob(train_df['question_text'][1]).words\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45954b9c239d2976241d36fae0e51f74507e6e71"},"cell_type":"code","source":"#Stemming\n'''Stemming refers to the removal of suffices, like “ing”, “ly”, “s”, etc. by a simple rule-based approach. \nFor this purpose, we will use PorterStemmer from the NLTK library.\n'''\n\nfrom nltk.stem import PorterStemmer\nst = PorterStemmer()\ntrain_df['question_text'][:10].apply(lambda x: \" \".join([st.stem(word) for word in x.split()]))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b1a61d5ae64a9a4db7aca9e4611ea3a090b6162"},"cell_type":"code","source":"#Lemmatization\n'''Lemmatization is a more effective option than stemming because it converts the word into its root word, rather than just stripping the suffices.\n'''\n\nfrom textblob import Word\ntrain_df['question_text'] = train_df['question_text'].apply(lambda x: \" \".join([Word(word).lemmatize() for word in x.split()]))\ntrain_df['question_text'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc0d7e2fa14d142dfcad50353b9b97a604fbec1f"},"cell_type":"code","source":"#N-grams\n'''N-grams are the combination of multiple words used together. Ngrams with N=1 are called unigrams. \nSimilarly, bigrams (N=2), trigrams (N=3) and so on can also be used. '''\n\nTextBlob(train_df['question_text'][0]).ngrams(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fbbaecf3911456514652208b4fc8ffb00acf661d"},"cell_type":"code","source":"#Term frequency\n'''Term frequency is simply the ratio of the count of a word present in a sentence, to the length of the sentence.\nTF = (Number of times term T appears in the particular row) / (number of terms in that row)'''\n\ntf1 = (train_df['question_text'][1:2]).apply(lambda x: pd.value_counts(x.split(\" \"))).sum(axis = 0).reset_index()\ntf1.columns = ['words','tf']\ntf1\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"21446d780408f9f58524ed013847790b6600f057"},"cell_type":"code","source":"#Inverse Document Frequency\n'''\nThe intuition behind inverse document frequency (IDF) is that a word is not of much use to us if it’s appearing in all the documents.\nThe IDF of each word is the log of the ratio of the total number of rows to the number of rows in which that word is present.\nIDF = log(N/n), where, N is the total number of rows and n is the number of rows in which the word was present.\n'''\n\nfor i,word in enumerate(tf1['words']):\n  tf1.loc[i, 'idf'] = np.log(train_df.shape[0]/(len(train_df[train_df['question_text'].str.contains(word)])))\n\ntf1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73a3557b2279643e510f40180e2dcb46317ee31b"},"cell_type":"code","source":"#Term Frequency – Inverse Document Frequency (TF-IDF)\ntf1['tfidf'] = tf1['tf'] * tf1['idf']\ntf1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7628f1377b9cc74f7bfc0c2c1081091ef04ba96"},"cell_type":"code","source":"#Using sklearn has a separate function to directly obtain TF and IDF \n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\ntfidf = TfidfVectorizer(max_features=1000, lowercase=True, analyzer='word',\n stop_words= 'english',ngram_range=(1,1))\ntrain_vect = tfidf.fit_transform(train_df['question_text'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"27c78a28f95dd3152f4bde5be80408b05b3f1381"},"cell_type":"code","source":"#Bag of Words\n'''\nBag of Words (BoW) refers to the representation of text which describes the presence of words within the text data. The intuition behind this is that two similar text fields will contain similar kind of words, and will therefore have a similar bag of words.\n'''\nfrom sklearn.feature_extraction.text import CountVectorizer\nbow = CountVectorizer(max_features=1000, lowercase=True, ngram_range=(1,1),analyzer = \"word\")\ntrain_bow = bow.fit_transform(train_df['question_text'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a49a7f97a9e630a25367f4d2254ac52f0e77773d"},"cell_type":"code","source":"#Hashing with HashingVectorizer\n'''Counts and frequencies can be very useful, but one limitation of these methods is that the vocabulary can become very large.\nThis, in turn, will require large vectors for encoding documents and impose large requirements on memory and slow down algorithms.\nA clever work around is to use a one way hash of words to convert them to integers. The clever part is that no vocabulary is required and you can choose an arbitrary-long fixed length vector. A downside is that the hash is a one-way function so there is no way to convert the encoding back to a word (which may not matter for many supervised learning tasks).\n'''\n\nfrom sklearn.feature_extraction.text import HashingVectorizer\n# create the transform\nvectorizer = HashingVectorizer(n_features=100)\nvector = vectorizer.fit_transform(train_df['question_text'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}