{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Getting Started with Sentiment Analysis"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport sqlite3\nimport pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport sklearn\nfrom sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer, CountVectorizer\nfrom sklearn.metrics import confusion_matrix, roc_curve, auc\nfrom sklearn import metrics\n\n\nimport re, string\nimport nltk\nfrom nltk.stem.porter import PorterStemmer\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\nfrom nltk.stem.wordnet import WordNetLemmatizer\nfrom gensim.models import Word2Vec, KeyedVectors\n\n\nimport pickle\nfrom tqdm import tqdm\nimport math as math\npd.set_option('display.max_rows', 500)\npd.set_option('display.max_columns', 500)\npd.set_option('display.width', 1000)\npd.set_option('precision', 5)\npd.options.display.float_format = '{:20,.2f}'.format\nnp.set_printoptions(suppress =True) \nprint(nltk.__version__)\nfrom gensim import __version__\nprint(__version__)\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"con = sqlite3.connect('/kaggle/input/amazon-fine-food-reviews/database.sqlite') \nfiltered_data = pd.read_sql_query(\"\"\" SELECT * FROM Reviews  WHERE Score != 3 LIMIT 50000\"\"\", con)\nfiltered_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"filtered_data.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in ['ProductId', 'UserId', 'ProfileName', 'Score']:\n    print('No of unique {} values : {}'.format(i,filtered_data[i].nunique()))\n    if i == 'Score':\n        print(filtered_data[i].value_counts())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Looks like there is a Class Imbalance between #+ve vs #-ve reviews. Appropriate Oversampling or Undersampling strategy must be tried"},{"metadata":{"trusted":true},"cell_type":"code","source":"#filtered_data['Score'].apply(lambda x:0 if x<3 else 1).head()\nfiltered_data['Score'] = filtered_data['Score'].apply(lambda x:'negative' if x<3 else 'positive')\nfiltered_data['Score'].value_counts()/len(filtered_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"query = \"\"\"\nSELECT UserId, ProductId, ProfileName, Time, Score, Text, COUNT(*) count_duplicate\nFROM Reviews\nGROUP BY UserId, ProfileName, Time, Text\nHAVING COUNT(*)>1\n\"\"\"\ndf_duplicates = pd.read_sql_query(query, con)\ndf_duplicates.sort_values(by='count_duplicate', ascending=False, inplace=True)\nprint(df_duplicates.shape)\ndf_duplicates.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Removing Duplicate reviews (Review gets Duplicated for each Product Attribute eg: each Color/Size of Shirt or each Flavour of Ice Cream etc)"},{"metadata":{"trusted":true},"cell_type":"code","source":"query = \"\"\"\nSELECT *\nFROM Reviews\nWHERE Score != 3 AND UserId=\"AR5J8UI46CURR\"\nORDER BY ProductID\n\"\"\" \npd.read_sql_query(query, con)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_sorted = filtered_data.sort_values('ProductId', axis=0, ascending=True, inplace=False, kind='quicksort', na_position='last')\ndf_deduplicated = df_sorted.drop_duplicates(subset={\"UserId\",\"ProfileName\",\"Time\",\"Text\"}, keep='first', inplace=False)\nprint(df_deduplicated.shape)\ndf_deduplicated[df_deduplicated.UserId == \"AR5J8UI46CURR\"]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### What % of rows were duplicates "},{"metadata":{"trusted":true},"cell_type":"code","source":"np.round((1.0-(df_deduplicated['Id'].size*1.0)/(filtered_data['Id'].size*1.0))*100, 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Checking if there are any anomolous rows with Helpful numerator(x) greater than Helpful Denominator (x+y) as it is imposible"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_deduplicated[df_deduplicated.HelpfulnessNumerator > df_deduplicated.HelpfulnessDenominator].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_deduplicated = df_deduplicated[df_deduplicated.HelpfulnessNumerator <= df_deduplicated.HelpfulnessDenominator]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Text data preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Checking random reviews\nsent_0, sent_1000, sent_1500, sent_4900 = [],[],[],[]\ndict_randomreview = {0:sent_0, 1000:sent_1000, 1500:sent_1500, 4900:sent_4900}\nfor key,value in dict_randomreview.items():\n    value = df_deduplicated['Text'].values[key]\n    print(value)\n    print('='*50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Removing urls from text python\n> https://stackoverflow.com/a/40823105/4084039"},{"metadata":{"trusted":true},"cell_type":"code","source":"#dict_randomreview = {0:sent_0, 1000:sent_1000, 1500:sent_1500, 4900:sent_4900}\nfor key,value in dict_randomreview.items():\n    value = df_deduplicated['Text'].values[key]\n    dict_randomreview[key] = re.sub(r\"http\\S+\", \"\", value)\n    print(dict_randomreview[key])\n    print('='*50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Removing all HTML tags from each element\n> https://stackoverflow.com/questions/16206380/python-beautifulsoup-how-to-remove-all-tags-from-an-element"},{"metadata":{"trusted":true},"cell_type":"code","source":"from bs4 import BeautifulSoup\n\n#dict_randomreview = {0:sent_0, 1000:sent_1000, 1500:sent_1500, 4900:sent_4900}\nfor key,value in dict_randomreview.items():\n    value = df_deduplicated['Text'].values[key]\n    value = re.sub(r\"http\\S+\", \"\", value)\n    soup = BeautifulSoup(value, 'lxml')\n    dict_randomreview[key] = soup.get_text()\n    #print(text)\n    print(dict_randomreview[key])\n    print('='*50)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Expanding Contractions\n> https://stackoverflow.com/questions/19790188/expanding-english-language-contractions-in-python/47091490#47091490"},{"metadata":{"trusted":true},"cell_type":"code","source":"contractions = { \n\"ain't\": \"am not / are not / is not / has not / have not\",\n\"aren't\": \"are not / am not\",\n\"can't\": \"cannot\",\n\"can't've\": \"cannot have\",\n\"'cause\": \"because\",\n\"could've\": \"could have\",\n\"couldn't\": \"could not\",\n\"couldn't've\": \"could not have\",\n\"didn't\": \"did not\",\n\"doesn't\": \"does not\",\n\"don't\": \"do not\",\n\"hadn't\": \"had not\",\n\"hadn't've\": \"had not have\",\n\"hasn't\": \"has not\",\n\"haven't\": \"have not\",\n\"he'd\": \"he had / he would\",\n\"he'd've\": \"he would have\",\n\"he'll\": \"he shall / he will\",\n\"he'll've\": \"he shall have / he will have\",\n\"he's\": \"he has / he is\",\n\"how'd\": \"how did\",\n\"how'd'y\": \"how do you\",\n\"how'll\": \"how will\",\n\"how's\": \"how has / how is / how does\",\n\"I'd\": \"I had / I would\",\n\"I'd've\": \"I would have\",\n\"I'll\": \"I shall / I will\",\n\"I'll've\": \"I shall have / I will have\",\n\"I'm\": \"I am\",\n\"I've\": \"I have\",\n\"isn't\": \"is not\",\n\"it'd\": \"it had / it would\",\n\"it'd've\": \"it would have\",\n\"it'll\": \"it shall / it will\",\n\"it'll've\": \"it shall have / it will have\",\n\"it's\": \"it has / it is\",\n\"let's\": \"let us\",\n\"ma'am\": \"madam\",\n\"mayn't\": \"may not\",\n\"might've\": \"might have\",\n\"mightn't\": \"might not\",\n\"mightn't've\": \"might not have\",\n\"must've\": \"must have\",\n\"mustn't\": \"must not\",\n\"mustn't've\": \"must not have\",\n\"needn't\": \"need not\",\n\"needn't've\": \"need not have\",\n\"o'clock\": \"of the clock\",\n\"oughtn't\": \"ought not\",\n\"oughtn't've\": \"ought not have\",\n\"shan't\": \"shall not\",\n\"sha'n't\": \"shall not\",\n\"shan't've\": \"shall not have\",\n\"she'd\": \"she had / she would\",\n\"she'd've\": \"she would have\",\n\"she'll\": \"she shall / she will\",\n\"she'll've\": \"she shall have / she will have\",\n\"she's\": \"she has / she is\",\n\"should've\": \"should have\",\n\"shouldn't\": \"should not\",\n\"shouldn't've\": \"should not have\",\n\"so've\": \"so have\",\n\"so's\": \"so as / so is\",\n\"that'd\": \"that would / that had\",\n\"that'd've\": \"that would have\",\n\"that's\": \"that has / that is\",\n\"there'd\": \"there had / there would\",\n\"there'd've\": \"there would have\",\n\"there's\": \"there has / there is\",\n\"they'd\": \"they had / they would\",\n\"they'd've\": \"they would have\",\n\"they'll\": \"they shall / they will\",\n\"they'll've\": \"they shall have / they will have\",\n\"they're\": \"they are\",\n\"they've\": \"they have\",\n\"to've\": \"to have\",\n\"wasn't\": \"was not\",\n\"we'd\": \"we had / we would\",\n\"we'd've\": \"we would have\",\n\"we'll\": \"we will\",\n\"we'll've\": \"we will have\",\n\"we're\": \"we are\",\n\"we've\": \"we have\",\n\"weren't\": \"were not\",\n\"what'll\": \"what shall / what will\",\n\"what'll've\": \"what shall have / what will have\",\n\"what're\": \"what are\",\n\"what's\": \"what has / what is\",\n\"what've\": \"what have\",\n\"when's\": \"when has / when is\",\n\"when've\": \"when have\",\n\"where'd\": \"where did\",\n\"where's\": \"where has / where is\",\n\"where've\": \"where have\",\n\"who'll\": \"who shall / who will\",\n\"who'll've\": \"who shall have / who will have\",\n\"who's\": \"who has / who is\",\n\"who've\": \"who have\",\n\"why's\": \"why has / why is\",\n\"why've\": \"why have\",\n\"will've\": \"will have\",\n\"won't\": \"will not\",\n\"won't've\": \"will not have\",\n\"would've\": \"would have\",\n\"wouldn't\": \"would not\",\n\"wouldn't've\": \"would not have\",\n\"y'all\": \"you all\",\n\"y'all'd\": \"you all would\",\n\"y'all'd've\": \"you all would have\",\n\"y'all're\": \"you all are\",\n\"y'all've\": \"you all have\",\n\"you'd\": \"you had / you would\",\n\"you'd've\": \"you would have\",\n\"you'll\": \"you shall / you will\",\n\"you'll've\": \"you shall have / you will have\",\n\"you're\": \"you are\",\n\"you've\": \"you have\"\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://stackoverflow.com/a/47091490/4084039\nimport re    \ndef decontracted(phrase):\n    # specific\n    phrase = re.sub(r\"won't\", \"will not\", phrase)\n    phrase = re.sub(r\"can\\'t\", \"can not\", phrase)\n\n    # general\n    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n    return phrase\n\n# def decontracted(phrase):\n#     for key,value in contractions.items():\n#         phrase = re.sub(key,value,phrase)\n#         return phrase","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for k,v in dict_randomreview.items():\n    print(k,v)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#dict_randomreview = {0:sent_0, 1000:sent_1000, 1500:sent_1500, 4900:sent_4900}\nsentence_1500 = decontracted(dict_randomreview[1500])\nprint(sentence_1500)\nprint(\"=\"*100)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Removing words that contain numbers in them: \n> https://stackoverflow.com/a/18082370/4084039"},{"metadata":{"trusted":true},"cell_type":"code","source":"#dict_randomreview = {0:sent_0, 1000:sent_1000, 1500:sent_1500, 4900:sent_4900}\nsentence_0 = decontracted(dict_randomreview[0])\nsentence_0 = re.sub(\"\\S*\\d\\S*\", \"\", sentence_0).strip()\nprint(sentence_0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Removing all Special characters \n> https://stackoverflow.com/a/5843547/4084039"},{"metadata":{"trusted":true},"cell_type":"code","source":"sentence_1500 = re.sub('[^A-Za-z0-9]+', ' ', decontracted(dict_randomreview[1500]))\nprint(sentence_1500)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Removing all the stop words like: 'no', 'nor', 'not', 'the', 'you', ....\n>  https://gist.github.com/sebleier/554280"},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.corpus import stopwords\nprint(stopwords.words(\"english\"))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords_list = set(['br', 'the', 'i', 'me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\",\n            \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', 'himself',\n            'she', \"she's\", 'her', 'hers', 'herself', 'it', \"it's\", 'its', 'itself', 'they', 'them', 'their',\n            'theirs', 'themselves', 'what', 'which', 'who', 'whom', 'this', 'that', \"that'll\", 'these', 'those',\n            'am', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 'have', 'has', 'had', 'having', 'do', 'does',\n            'did', 'doing', 'a', 'an', 'the', 'and', 'but', 'if', 'or', 'because', 'as', 'until', 'while', 'of',\n            'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', 'through', 'during', 'before', 'after',\n            'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off', 'over', 'under', 'again', 'further',\n            'then', 'once', 'here', 'there', 'when', 'where', 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more',\n            'most', 'other', 'some', 'such', 'only', 'own', 'same', 'so', 'than', 'too', 'very',\n            's', 't', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', 'll', 'm', 'o', 're',\n            've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn', \"didn't\", 'doesn', \"doesn't\", 'hadn',\n            \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn',\n            \"mustn't\", 'needn', \"needn't\", 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\",\n            'won', \"won't\", 'wouldn', \"wouldn't\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Applying all the above cleaning to the entire dataframe"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\npreprocessed_reviews = []\n# tqdm is for printing the status bar\nfor sentance in tqdm(df_deduplicated['Text'].values):\n    sentance = re.sub(r\"http\\S+\", \"\", sentance)\n    sentance = BeautifulSoup(sentance, 'lxml').get_text()\n    sentance = decontracted(sentance)\n    sentance = re.sub(\"\\S*\\d\\S*\", \"\", sentance).strip()\n    sentance = re.sub('[^A-Za-z]+', ' ', sentance)\n    # https://gist.github.com/sebleier/554280\n    sentance = ' '.join(e.lower() for e in sentance.split() if e.lower() not in stopwords_list)\n    preprocessed_reviews.append(sentance.strip())\n    \nprint(preprocessed_reviews[1500])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Feature Engineering on Text data "},{"metadata":{"trusted":true},"cell_type":"code","source":"[w for w in dir(sklearn.feature_extraction.text) if not w.startswith('_')]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### (BoW) Bag of Words - simply put is ----------> df.words.value_counts()\n> https://en.wikipedia.org/wiki/Bag-of-words_model <br/>\n> https://stackabuse.com/python-for-nlp-creating-bag-of-words-model-from-scratch/ <br/>\n> http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.453.5924&rep=rep1&type=pdf <br/>\n> https://www.youtube.com/watch?v=IRKDrrzh4dE <br/>\n> https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html <br/>\n> <br/>"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\n\ncount_vect = CountVectorizer().fit(preprocessed_reviews)\nprint(\"some random words/features : \", count_vect.get_feature_names()[:10])\nprint('='*50)\n\nword_count = count_vect.transform(preprocessed_reviews)\nprint(\"the type of count vectorizer \",type(word_count))\nprint(\"the shape of out text BOW vectorizer \",word_count.get_shape())\nprint(\"the number of unique words \", word_count.get_shape()[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"> Bag of Words is Unigram based(only 1 word) and hence discards Sequential information of the data. "},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Bi-grams and n-Grams simply put is ----------> Convert 2(or n) sequential words into one word (vector representation)\n> https://kavita-ganesan.com/what-are-n-grams/ <br/>\n> https://en.wikipedia.org/wiki/Bigram <br/>\n> https://en.wikipedia.org/wiki/N-gram <br/>\n> https://web.stanford.edu/~jurafsky/slp3/3.pdf <br/>\n> http://l2r.cs.uiuc.edu/~danr/Teaching/CS598-05/Papers/Church-ngrams.pdf <br/>\n> https://lagunita.stanford.edu/c4x/Engineering/CS-224N/asset/slp4.pdf <br/>\n> https://people.cs.umass.edu/~mccallum/papers/tng-icdm07.pdf <br/>\n> https://catalog.ldc.upenn.edu/LDC2006T13 <br/>\n> https://www.youtube.com/watch?v=E_mN90TYnlg <br/>\n> https://scikit-learn.org/stable/modules/generated/sklearn.feature_extraction.text.CountVectorizer.html#sklearn.feature_extraction.text.CountVectorizer <br/>\n> <br/>"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nCountVectorizer()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### note the arguements ngram_range, min_df, max_df\n> ngram_range --> Strictly Unigram/Bigram/Trigram, all 3 unigram+bigram+trigram included, ... --> (1,1);(2,2);(3,3);(1,3) <br/>\n> min_df --> Minimum Word Count allowed (threshold) <br/>\n> max_df --> Maximum Word Count allowed (threshold) <br/>"},{"metadata":{"trusted":true},"cell_type":"code","source":"count_vect = CountVectorizer(ngram_range=(1,2), min_df=10, max_df=5000)\nbigram_wordcounts = count_vect.fit_transform(preprocessed_reviews)\nprint(\"the type of count vectorizer \",type(bigram_wordcounts))\nprint(\"the shape of out text BOW vectorizer \",bigram_wordcounts.get_shape())\nprint(\"the number of unique words including both unigrams and bigrams \", bigram_wordcounts.get_shape()[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### tf-idf (Term frequency - inverse document frequency)\n\n> https://en.wikipedia.org/wiki/Tf%E2%80%93idf <br/>\n> https://www.kdnuggets.com/2018/08/wtf-tf-idf.html <br/>\n> https://www.researchgate.net/publication/220387577_A_probabilistic_justification_for_using_tfidf_term_weighting_in_information_retrieval <br/>\n> https://ccc.inaoep.mx/~villasen/index_archivos/cursoTL/articulos/Aizawa-tf-idfMeasures.pdf <br/>\n> https://www.scss.tcd.ie/khurshid.ahmad/Research/Sentiments/tfidf_relevance.pdf <br/>\n> https://www.semanticscholar.org/topic/Tf%E2%80%93idf/72426 <br/>\n> http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.438.2284&rep=rep1&type=pdf <br/>\n> http://citeseerx.ist.psu.edu/viewdoc/download?doi=10.1.1.121.1424&rep=rep1&type=pdf <br/>\n> http://ecsjournal.org/Archive/Volume42/Issue3/5.pdf <br/>\n> https://www.youtube.com/watch?v=6HuKFh0BatQ <br/>\n> https://www.youtube.com/watch?v=C25txE_dq90 <br/>\n>  <br/>"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfTransformer, TfidfVectorizer\nfor i in [TfidfTransformer, TfidfVectorizer]:\n    print([w for w in dir(i) if not w.startswith('_')])\n    print('='*50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import inspect\nprint(inspect.getargspec(TfidfVectorizer))\nprint('='*50)\nprint(inspect.getargspec(TfidfTransformer))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf_idf_vect = TfidfVectorizer(ngram_range=(1,2), min_df=10)\ntf_idf_vect.fit(preprocessed_reviews)\nprint(\"some sample features \",tf_idf_vect.get_feature_names()[0:10])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf_idf_forinput = tf_idf_vect.transform(preprocessed_reviews)\nprint(\"the type of count vectorizer \",type(tf_idf_forinput))\nprint(\"the shape of out text TFIDF vectorizer \",tf_idf_forinput.get_shape())\nprint(\"the number of unique words including both unigrams and bigrams \", tf_idf_forinput.get_shape()[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"> tf-idf still doesn't take synonyms/ almost similar words into considerations eg: tasty = delicious, cheap = affordable"},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### Word2Vec\n\n> https://en.wikipedia.org/wiki/Word2vec <br/>\n> https://papers.nips.cc/paper/5021-distributed-representations-of-words-and-phrases-and-their-compositionality.pdf <br/>\n> https://arxiv.org/pdf/1301.3781.pdf <br/>\n> https://www.researchgate.net/publication/281812760_TwoToo_Simple_Adaptations_of_Word2Vec_for_Syntax_Problems <br/>\n> http://jalammar.github.io/illustrated-word2vec/ <br/>\n> https://www.researchgate.net/publication/321709086_How_Does_Word2Vec_Work <br/>\n> https://arxiv.org/vc/arxiv/papers/1603/1603.04259v2.pdf <br/>\n> https://www.academia.edu/33141616/Novel2Vec_Characterising_19th_Century_Fiction_via_Word_Embeddings <br/>\n> https://arxiv.org/pdf/1310.4546.pdf <br/>\n> https://www.microsoft.com/en-us/research/wp-content/uploads/2016/02/rvecs.pdf <br/>\n> https://en.wikipedia.org/wiki/Vector_quantization <br/>\n> http://www.ws.binghamton.edu/fowler/fowler%20personal%20page/EE523_files/Ch_10_1%20VQ%20Description%20(PPT).pdf <br/>\n> https://www.youtube.com/watch?v=5PL0TmQhItY <br/>\n> https://www.youtube.com/watch?v=ERibwqs9p38 <br/>\n> https://www.tensorflow.org/tutorials/text/word_embeddings <br/>\n> <br/>"},{"metadata":{"trusted":true},"cell_type":"code","source":"list_of_sentence = []\nfor sentence in tqdm(preprocessed_reviews):\n    list_of_sentence.append(sentence.split())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Training your own Word2Vec"},{"metadata":{"trusted":true},"cell_type":"code","source":"import gensim\nprint([w for w in dir(gensim.models) if not w.startswith('_')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from gensim.models import Word2Vec\nprint(inspect.getargspec(Word2Vec))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word2vec_model = Word2Vec(list_of_sentence, min_count=5, size=50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print([w for w in dir(word2vec_model) if not w.startswith('_')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print([w for w in dir(word2vec_model.wv) if not w.startswith('_')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word2vec_words = list(word2vec_model.wv.vocab)\nprint(\"number of words that occured minimum 5 times \",len(word2vec_words))\nprint(\"sample words \", word2vec_words[0:50])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Using Google's New Vector Word2Vec\n> https://radimrehurek.com/gensim/models/keyedvectors.html"},{"metadata":{"trusted":true},"cell_type":"code","source":"# from gensim.models import KeyedVectors\n# print(inspect.getargspec(KeyedVectors))\n# print([w for w in dir(KeyedVectors) if not w.startswith('_')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# filepath = '/kaggle/input/quora-insincere-questions-classification/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n\n# embeddings_index = {}\n# Google_Word2Vec = KeyedVectors.load_word2vec_format(filepath, binary=True)\n# print([w for w in dir(Google_Word2Vec)])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# for word,vector in zip(Google_Word2Vec.vocab, Google_Word2Vec.vectors):\n#     coefs = np.asarray(vector, dtype='float32')\n#     embeddings_index[word] = coefs\n# Google_word2vec_words = list(Google_Word2Vec.wv.vocab)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# test_word = 'Movie'\n# print(len(embeddings_index[test_word]))\n# embeddings_index[test_word]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Compare results of Our Own Word2Vec trained on input data vs. Google News Word2Vec Example 1"},{"metadata":{},"cell_type":"markdown","source":"### Our Own Word2Vec "},{"metadata":{"trusted":true},"cell_type":"code","source":"word2vec_model.wv.most_similar('great')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Google's Word2Vec "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Google_Word2Vec.wv.most_similar('great')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Compare results of Our Own Word2Vec trained on input data vs. Google News Word2Vec Example 2"},{"metadata":{},"cell_type":"markdown","source":"### Our Own Word2Vec "},{"metadata":{"trusted":true},"cell_type":"code","source":"word2vec_model.wv.most_similar('worst')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Google's Word2Vec "},{"metadata":{"trusted":true},"cell_type":"code","source":"# Google_Word2Vec.wv.most_similar('worst')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Converting Reviews/Sequence of Words into Vector using Average Word2Vec"},{"metadata":{"trusted":true},"cell_type":"code","source":"sent_vectors = [];\nfor sent in tqdm(list_of_sentence): # for each review/sentence\n    sent_vec = np.zeros(50) # as word vectors are of zero length 50, you might need to change this to 300 if you use google's w2v\n    cnt_words =0; # num of words with a valid vector in the sentence/review\n    for word in sent: # for each word in a review/sentence\n        if word in word2vec_words:\n            vec = word2vec_model.wv[word]\n            sent_vec += vec\n            cnt_words += 1\n    if cnt_words != 0:\n        sent_vec /= cnt_words\n    sent_vectors.append(sent_vec)\nprint(len(sent_vectors))\nprint(len(sent_vectors[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Google_sent_vectors = [];\n# for sent in tqdm(list_of_sentence): # for each review/sentence\n#     sent_vec = np.zeros(300) # as word vectors are of zero length 300 for google's w2v\n#     cnt_words =0; \n#     for word in sent:\n#         if word in Google_word2vec_words:\n#             vec = Google_Word2Vec.wv[word]\n#             sent_vec += vec\n#             cnt_words += 1\n#     if cnt_words != 0:\n#         sent_vec /= cnt_words\n#     Google_sent_vectors.append(sent_vec)\n# print(len(Google_sent_vectors))\n# print(len(Google_sent_vectors[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# ML Modelling Phase"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### This is just the first draft version. Will include some more of my own code with lots of updates in coming weeks"},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"","_uuid":"","trusted":true},"cell_type":"code","source":"# import pandas as pd\n# Reviews = pd.read_csv(\"../input/amazon-fine-food-reviews/Reviews.csv\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}