{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport sqlite3\nimport pandas as pd\nimport numpy as np\nimport nltk\nimport string\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.feature_extraction.text import TfidfTransformer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.metrics import confusion_matrix\nfrom sklearn import metrics\nfrom sklearn.metrics import roc_curve, auc\nfrom nltk.stem.porter import PorterStemmer\n\nimport re\n# Tutorial about Python regular expressions: https://pymotw.com/2/re/\nimport string\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\n\n\n\n\n# using the SQLite Table to read data.\ncon = sqlite3.connect('../input/amazon-fine-food-reviews/database.sqlite') \n\n\n\n#filtering only positive and negative reviews i.e. \n# not taking into consideration those reviews with Score=3\nfiltered_data = pd.read_sql_query(\"\"\" SELECT * FROM Reviews WHERE Score != 3 \"\"\", con) \n\n  \n# Give reviews with Score>3 a positive rating, and reviews with a score<3 a negative rating.\ndef partition(x):\n    if x < 3:\n        return 'negative'\n    return 'positive'\n\n#changing reviews with score less than 3 to be positive and vice-versa\nactualScore = filtered_data['Score']\npositiveNegative = actualScore.map(partition) \nfiltered_data['Score'] = positiveNegative","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d66a9b19bab66dc5cb9a81ad4743d3e5405e67c2"},"cell_type":"code","source":"import datetime\n\nfiltered_data[\"Time\"] = filtered_data[\"Time\"].map(lambda t: datetime.datetime.fromtimestamp(int(t)).strftime('%Y-%m-%d %H:%M:%S'))\n\nsortedData = filtered_data.sort_values('ProductId',axis=0,kind=\"quicksort\", ascending=True)\nfinal = sortedData.drop_duplicates(subset={\"UserId\",\"ProfileName\",\"Time\",\"Text\"},keep=\"first\",inplace=False)\n\nfinal = final[final.HelpfulnessNumerator <= final.HelpfulnessDenominator]\n\n#As data is huge, due to computation limitation we will randomly select data. we will try to pick data in a way so that it doesn't make data imbalance problem\nfinalp = final[final.Score == 'positive']\nfinalp = finalp.sample(frac=0.035,random_state=1) #0.055\n\nfinaln = final[final.Score == 'negative']\nfinaln = finaln.sample(frac=0.15,random_state=1) #0.25\n\nfinal = pd.concat([finalp,finaln],axis=0)\n\n#sording data by timestamp so that it can be devided in train and test dataset for time based slicing.\nfinal = final.sort_values('Time',axis=0,kind=\"quicksort\", ascending=True).reset_index(drop=True)\n\n\nprint(final.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"69f1d75eca87b6118cd753547b18d3a03feb8d0d"},"cell_type":"code","source":"#Checking to see how much % of data still remains\n(final['Id'].size*1.0)/(filtered_data['Id'].size*1.0)*100\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3d729730046a60ee3f132551b4706d0614a3e92"},"cell_type":"code","source":"from nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\nfrom nltk.stem.wordnet import WordNetLemmatizer\n\n\n\nstop = set(stopwords.words('english')) #set of stopwords\nsno = nltk.stem.SnowballStemmer('english') #initialising the snowball stemmer\n\ndef cleanhtml(sentence): #function to clean the word of any html-tags\n    cleanr = re.compile('<.*?>')\n    cleantext = re.sub(cleanr, ' ', sentence)\n    return cleantext\ndef cleanpunc(sentence): #function to clean the word of any punctuation or special characters\n    cleaned = re.sub(r'[?|!|\\'|\"|#]',r'',sentence)\n    cleaned = re.sub(r'[.|,|)|(|\\|/]',r' ',cleaned)\n    return  cleaned\nprint(stop)\nprint('************************************')\nprint(sno.stem('tasty'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25482bf66fc53ec17ec9fd33f1829318c77b279e"},"cell_type":"code","source":"final_postive=final[final['Score']=='positive']\nfinal_negative=final[final['Score']=='negative']\n\n\n\nfinal_postive=final_postive.replace('positive',1)\nfinal_negative=final_negative.replace('negative',0)\n\nfinal_postiv=final_postive.iloc[0:2000, :]\nfinal_negativ=final_negative.iloc[0:2000, :]\n\n\n#final_negative.head()\nfinal_4000=(final_postiv,final_negativ)\nfinal_4000=pd.concat(final_4000)\ntarget=final_4000['Score']\nfinal_4000=final_4000.drop('Score',axis=1)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"849ef798d32b9faf1f8db11b358a1a96b2b237ca"},"cell_type":"code","source":"#Code for implementing step-by-step the checks mentioned in the pre-processing phase\n# this code takes a while to run as it needs to run on 500k sentences.\ni=0\nstr1=' '\nfinal_string=[]\nall_positive_words=[] # store words from +ve reviews here\nall_negative_words=[] # store words from -ve reviews here.\ns=''\nfor sent in final['Text'].values:\n    filtered_sentence=[]\n    #print(sent);\n    sent=cleanhtml(sent) # remove HTMl tags\n    for w in sent.split():\n        for cleaned_words in cleanpunc(w).split():\n            if((cleaned_words.isalpha()) & (len(cleaned_words)>2)):    \n                if(cleaned_words.lower() not in stop):\n                    s=(sno.stem(cleaned_words.lower())).encode('utf8')\n                    filtered_sentence.append(s)\n                    if (final['Score'].values)[i] == 'positive': \n                        all_positive_words.append(s) #list of all words used to describe positive reviews\n                    if(final['Score'].values)[i] == 'negative':\n                        all_negative_words.append(s) #list of all words used to describe negative reviews reviews\n                else:\n                    continue\n            else:\n                continue \n    #print(filtered_sentence)\n    str1 = b\" \".join(filtered_sentence) #final string of cleaned words\n    #print(\"***********************************************************************\")\n    \n    final_string.append(str1)\n    i+=1\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"67ebd22f19600d5fcd939ca9919f74c1da72dc20"},"cell_type":"code","source":"final['CleanedText']=final_string #adding a column of CleanedText which displays the data after pre-processing of the review \nfinal['CleanedText']=final['CleanedText'].str.decode(\"utf-8\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e628bf541fcd909ee724383953258a1e474eeb9f"},"cell_type":"code","source":"# store final table into an SQlLite table for future.\nconn = sqlite3.connect('final.sqlite')\nc=conn.cursor()\nconn.text_factory = str\nfinal.to_sql('Reviews', conn,  schema=None, if_exists='replace', index=True, index_label=None, chunksize=None, dtype=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3ddbb92e5d1472a0cf15447f621a56d896346199"},"cell_type":"code","source":"len(final)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bbaaa40f866cc73fc5d46c1bf6ac0c4abfd2d52b"},"cell_type":"code","source":"#BoW\ncount_vect = CountVectorizer() #in scikit-learn\nfinal_counts = count_vect.fit_transform(final['CleanedText'].values)\nprint(\"the type of count vectorizer \",type(final_counts))\nprint(\"the shape of out text BOW vectorizer \",final_counts.get_shape())\nprint(\"the number of unique words \", final_counts.get_shape()[1])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51b5afd6285bac35647ab9e6469637f6db425228"},"cell_type":"code","source":"final_counts.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8542db3fd35987ea74c291e1c6e7e2046a1d903"},"cell_type":"code","source":"from sklearn.preprocessing import StandardScaler\n\nfinal_bow_np = StandardScaler(with_mean=False).fit_transform(final_counts)\nfinal.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d12f8308542a81e6052bff56422be1e104c1aa61"},"cell_type":"code","source":"final_bow_np.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e0a267e2caa56fe8515f4cf13f253137824627a"},"cell_type":"code","source":"#We already have sorted data by timestamp so we will use first 70% of data as Train with cross validation and next 30% for test\nimport math\nfrom sklearn import datasets\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import train_test_split\n\nX = final_bow_np\ny = final['Score']\n\nX_train =  final_bow_np[:math.ceil(len(final)*.7)] \nX_test = final_bow_np[math.ceil(len(final)*.7):]\ny_train = y[:math.ceil(len(final)*.7)]\ny_test =  y[math.ceil(len(final)*.7):]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"788efffdaa064ef7828ba896f58897e3822c1cd7"},"cell_type":"code","source":"X_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d66eba34e39027cbe32cd14a55ed9b851cac3b37"},"cell_type":"code","source":"from sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.model_selection import train_test_split\nimport numpy as np\nimport math\n#import numpy.reshape\n#dense=final_counts.toarray()\n\n#np.array(y_train)\nX_train =  final_counts[:math.ceil(len(final)*.6)] \nX_cv = final_counts[round(len(final)*.6):round(len(final)*.8)]\nX_test = final_counts[round(len(final)*.8):]\n\ny_train = y[:round(len(final)*.6)]\ny_cv =  y[round(len(final)*.6):round(len(final)*.8)]\ny_test = y[round(len(final)*.8):]\n\n#knn=KNeighborsClassifier(n_neighbors=5)\n#knn.fit(X_train,y_train)\n#knn.predict(X_test)\n#print(knn.score(X_test,y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5575054121126157d4886e77e6cfff3cdf3b9428"},"cell_type":"code","source":"print(X_train.shape)\nprint(X_test.shape)\nprint(y_train.shape)\nprint(y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f92d0259d010838868a1d5a96ec5562df329a9ee"},"cell_type":"code","source":"X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"28ebcd36dd51662bdf5c75471ba76cffea7656fb"},"cell_type":"code","source":"# ============================== loading libraries ===========================================\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nfrom sklearn.cross_validation import train_test_split\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.cross_validation import cross_val_score\nfrom collections import Counter\nfrom sklearn.metrics import accuracy_score\nfrom sklearn import cross_validation\n# =============================================================================================","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"afc4cab52fd965315a6d36751d4773cbbb9579e2"},"cell_type":"code","source":"# split the data set into train and test\n#X_train,X_test,y_train,y_test=train_test_split(final_counts,target,test_size=0.3,random_state=42)\n#X_1, X_test, y_1, y_test = cross_validation.train_test_split(X, y, test_size=0.3, random_state=0)\n\n# split the train data set into cross validation train and cross validation test\n#X_tr, X_cv, y_tr, y_cv = cross_validation.train_test_split(X_train, y_train, test_size=0.3)\n\nfor i in range(1,30,2):\n    # instantiate learning model (k = 30)\n    knn = KNeighborsClassifier(n_neighbors=i)\n\n    # fitting the model on crossvalidation train\n    knn.fit(X_train, y_train)\n\n    # predict the response on the crossvalidation train\n    pred = knn.predict(X_cv)\n\n    # evaluate CV accuracy\n    acc = accuracy_score(y_cv, pred, normalize=True) * float(100)\n    print('\\nCV accuracy for k = %d is %d%%' % (i, acc))\n    \nknn = KNeighborsClassifier(1)\nknn.fit(X_train,y_train)\npred = knn.predict(X_test)\nacc = accuracy_score(y_test, pred, normalize=True) * float(100)\nprint('\\n****Test accuracy for k = 1 is %d%%' % (acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d1e553eab9885f3c5c4a35c558d73123edfd1c3"},"cell_type":"code","source":"\n\n# creating odd list of K for KNN\nmyList = list(range(0,50))\nneighbors = list(filter(lambda x: x % 2 != 0, myList))\n\n# empty list that will hold cv scores\ncv_scores = []\n\n# perform 10-fold cross validation\nfor k in neighbors:\n    knn = KNeighborsClassifier(n_neighbors=k)\n    scores = cross_val_score(knn, X_train, y_train, cv=10, scoring='accuracy')\n    cv_scores.append(scores.mean())\n\n# changing to misclassification error\nMSE = [1 - x for x in cv_scores]\n\n# determining best k\noptimal_k = neighbors[MSE.index(min(MSE))]\nprint('\\nThe optimal number of neighbors is %d.' % optimal_k)\n\n# plot misclassification error vs k \nplt.plot(neighbors, MSE)\n\nfor xy in zip(neighbors, np.round(MSE,3)):\n    plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')\n\nplt.xlabel('Number of Neighbors K')\nplt.ylabel('Misclassification Error')\nplt.show()\n\nprint(\"the misclassification error for each k value is : \", np.round(MSE,3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d5cf94cbc61fe389b5994b2b6380ceb9b1cc8b85"},"cell_type":"code","source":"# ============================== KNN with k = optimal_k ===============================================\n# instantiate learning model k = optimal_k\nknn_optimal = KNeighborsClassifier(n_neighbors=optimal_k)\n\n# fitting the model\nknn_optimal.fit(X_train, y_train)\n\n# predict the response\npred = knn_optimal.predict(X_test)\n\n# evaluate accuracy\nacc = accuracy_score(y_test, pred) * 100\nprint('\\nThe accuracy of the knn classifier for k = %d is %f%%' % (optimal_k, acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"059283912c6c4f960ec477ac793e45b6ea1cb874"},"cell_type":"code","source":"#bi-gram, tri-gram and n-gram\n\n#removing stop words like \"not\" should be avoided before building n-grams\ncount_vect = CountVectorizer(ngram_range=(1,2) ) #in scikit-learn\nfinal_bigram_counts = count_vect.fit_transform(final['CleanedText'].values)\nprint(\"the type of count vectorizer \",type(final_bigram_counts))\nprint(\"the shape of out text BOW vectorizer \",final_bigram_counts.get_shape())\nprint(\"the number of unique words including both unigrams and bigrams \", final_bigram_counts.get_shape()[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3b02e063fa3c2b6741ae75e2177cb6271f8b8146","scrolled":true},"cell_type":"code","source":"#Tfidf\ntf_idf_vect = TfidfVectorizer(ngram_range=(1,2))\nfinal_tf_idf = tf_idf_vect.fit_transform(final['CleanedText'].values)\nprint(\"the type of count vectorizer \",type(final_tf_idf))\nprint(\"the shape of out text TFIDF vectorizer \",final_tf_idf.get_shape())\nprint(\"the number of unique words including both unigrams and bigrams \", final_tf_idf.get_shape()[1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e09f5106d59f732481a3c4821c27143f4854346"},"cell_type":"code","source":"\nX_train1 =  final_tf_idf[:math.ceil(len(final)*.6)] \nX_cv1 = final_tf_idf[round(len(final)*.6):round(len(final)*.8)]\nX_test1 = final_tf_idf[round(len(final)*.8):]\n\ny_train1 = y[:round(len(final)*.6)]\ny_cv1 =  y[round(len(final)*.6):round(len(final)*.8)]\ny_test1 = y[round(len(final)*.8):]\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e181d23770c76d428ef59a4082c4272cd3c0d17"},"cell_type":"code","source":"# split the data set into train and test\n#X_train1,X_test1,y_train1,y_test1=train_test_split(final_tf_idf,target,test_size=0.3,random_state=42)\n#X_1, X_test, y_1, y_test = cross_validation.train_test_split(X, y, test_size=0.3, random_state=0)\n\n# split the train data set into cross validation train and cross validation test\n#X_train1, X_cv1, y_train1, y_cv1 = cross_validation.train_test_split(X_train1, y_train1, test_size=0.3)\n\nfor i in range(1,30,2):\n    # instantiate learning model (k = 30)\n    knn = KNeighborsClassifier(n_neighbors=i,algorithm='brute')\n\n    # fitting the model on crossvalidation train\n    knn.fit(X_train1, y_train1)\n\n    # predict the response on the crossvalidation train\n    pred = knn.predict(X_cv1)\n\n    # evaluate CV accuracy\n    acc = accuracy_score(y_cv1, pred, normalize=True) * float(100)\n    print('\\nCV accuracy for k = %d is %d%%' % (i, acc))\n    \nknn = KNeighborsClassifier(1,algorithm='brute')\nknn.fit(X_train,y_train)\npred = knn.predict(X_test)\nacc = accuracy_score(y_test, pred, normalize=True) * float(100)\nprint('\\n****Test accuracy for k = 1 is %d%%' % (acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"423d3bbe8b462a1df48b0ec434f6b449dde29b35"},"cell_type":"code","source":"\n\n# creating odd list of K for KNN\nmyList = list(range(0,50))\nneighbors = list(filter(lambda x: x % 2 != 0, myList))\n\n# empty list that will hold cv scores\ncv_scores = []\n\n# perform 10-fold cross validation\nfor k in neighbors:\n    knn = KNeighborsClassifier(n_neighbors=k,algorithm='brute')\n    scores = cross_val_score(knn, X_train1, y_train1, cv=10, scoring='accuracy')\n    cv_scores.append(scores.mean())\n\n# changing to misclassification error\nMSE = [1 - x for x in cv_scores]\n\n# determining best k\noptimal_k = neighbors[MSE.index(min(MSE))]\nprint('\\nThe optimal number of neighbors is %d.' % optimal_k)\n\n# plot misclassification error vs k \nplt.plot(neighbors, MSE)\n\nfor xy in zip(neighbors, np.round(MSE,3)):\n    plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')\n\nplt.xlabel('Number of Neighbors K')\nplt.ylabel('Misclassification Error')\nplt.show()\n\nprint(\"the misclassification error for each k value is : \", np.round(MSE,3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cb167df376409f9ae0aa28df1b5da28b9b11812f"},"cell_type":"code","source":"# ============================== KNN with k = optimal_k ===============================================\n# instantiate learning model k = optimal_k\nknn_optimal = KNeighborsClassifier(n_neighbors=optimal_k,algorithm='brute')\n\n# fitting the model\nknn_optimal.fit(X_train1, y_train1)\n\n# predict the response\npred = knn_optimal.predict(X_test1)\n\n# evaluate accuracy\nacc = accuracy_score(y_test1, pred) * 100\nprint('\\nThe accuracy of the knn classifier for k = %d is %f%%' % (optimal_k, acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c4ea45c2b5c70bc8e6e80f0bb8857d2e522bffa9"},"cell_type":"code","source":"features = tf_idf_vect.get_feature_names()\nprint(\"some sample features(unique words in the corpus)\",features[100000:100010])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"55afe2a298aae8de8e534ff157bd1ad408af26a8"},"cell_type":"code","source":"# source: https://buhrmann.github.io/tfidf-analysis.html\ndef top_tfidf_feats(row, features, top_n=25):\n    ''' Get top n tfidf values in row and return them with their corresponding feature names.'''\n    topn_ids = np.argsort(row)[::-1][:top_n]\n    top_feats = [(features[i], row[i]) for i in topn_ids]\n    df = pd.DataFrame(top_feats)\n    df.columns = ['feature', 'tfidf']\n    return df\n\ntop_tfidf = top_tfidf_feats(final_tf_idf[1,:].toarray()[0],features,25)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc5a267c7d71b590ae819cdbe4e22cc19283eb74"},"cell_type":"code","source":"top_tfidf","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91a61148873d41a54780b50cb865f5f79c675fe5"},"cell_type":"code","source":"# Train your own Word2Vec model using your own text corpus\ni=0\nlist_of_sent=[]\nfor sent in final['CleanedText'].values:\n    list_of_sent.append(sent.split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5dda3050f7cf12e00ed1013231da5fe5f1bb4e26"},"cell_type":"code","source":"print(final['CleanedText'].values[0])\nprint(\"*****************************************************************\")\nprint(list_of_sent[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5189b34a5f2b4e9f0d1079ea5860a6539aea9a9"},"cell_type":"code","source":"from gensim.models import Word2Vec\n# min_count = 5 considers only words that occured atleast 5 times\nw2v_model=Word2Vec(list_of_sent,min_count=5,size=50, workers=4)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fd8b749776ef675559e4277a009f182d063eca1c"},"cell_type":"code","source":"w2v_words = list(w2v_model.wv.vocab)\nprint(\"number of words that occured minimum 5 times \",len(w2v_words))\nprint(\"sample words \", w2v_words[0:50])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9458d74c1bf2bbb2bd4ad0f037ce3593aa838c7e"},"cell_type":"code","source":"w2v_model.wv.most_similar('tasti')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d3b3d25e7ffdfa72d9d038ff6f77bbe2bd4ff398"},"cell_type":"code","source":"w2v_model.wv.most_similar('like')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fbfbc833754d1c7e6c79dbc2f25634e621f9f4f"},"cell_type":"code","source":"count_vect_feat = count_vect.get_feature_names() # list of words in the BoW\nprint(count_vect_feat[count_vect_feat.index('like')])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ec4049dfd89b9d317f37fddf688a7992e2977f7"},"cell_type":"code","source":"i=0\nlist_of_sent=[]\nfor sent in final['CleanedText'].values:\n    list_of_sent.append(sent.split())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02ac69e04f5342e71c8ff6ef0c20461cc730b528"},"cell_type":"code","source":"print(final['CleanedText'].values[0])\nprint(\"*****************************************************************\")\nprint(list_of_sent[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8171e437e0272efeebafaaa55d3a5232e29302db"},"cell_type":"code","source":"# min_count = 5 considers only words that occured atleast 5 times\nw2v_model=Word2Vec(list_of_sent,min_count=5,size=50, workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ce547be14185efb0b929ea493dd4a79b67591b2"},"cell_type":"code","source":"w2v_words = list(w2v_model.wv.vocab)\nprint(\"number of words that occured minimum 5 times \",len(w2v_words))\nprint(\"sample words \", w2v_words[0:50])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d60c6917e1a80684babeb27c08bc8edd1c2c7dac"},"cell_type":"code","source":"count_vect_feat = count_vect.get_feature_names() # list of words in the BoW\nprint(count_vect_feat[count_vect_feat.index('like')])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c2292214f1cd07dcaf84178d7e4e0c8b44c910e8"},"cell_type":"markdown","source":"##avg_w2vec****"},{"metadata":{"trusted":true,"_uuid":"19494aedb1d5070daaade26b9d315a2d09497cca"},"cell_type":"code","source":"sent_vectors = []; # the avg-w2v for each sentence/review is stored in this list\nfor sent in list_of_sent: # for each review/sentence\n    sent_vec = np.zeros(50) # as word vectors are of zero length\n    cnt_words =0; # num of words with a valid vector in the sentence/review\n    for word in sent: # for each word in a review/sentence\n        if word in w2v_words:\n            vec = w2v_model.wv[word]\n            sent_vec += vec\n            cnt_words += 1\n    if cnt_words != 0:\n        sent_vec /= cnt_words\n    sent_vectors.append(sent_vec)\nprint(len(sent_vectors))\nprint(len(sent_vectors[0]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0fdc9ad396205734f549d3dd274a972981339af1"},"cell_type":"code","source":"# split the data set into train and test\n\n\nX_train3 =  sent_vectors[:math.ceil(len(final)*.6)] \nX_cv3 = sent_vectors[round(len(final)*.6):round(len(final)*.8)]\nX_test3 = sent_vectors[round(len(final)*.8):]\n\ny_train3 = y[:round(len(final)*.6)]\ny_cv3 =  y[round(len(final)*.6):round(len(final)*.8)]\ny_test3 = y[round(len(final)*.8):]\n\n\n#X_train3,X_test3,y_train3,y_test3 = train_test_split(sent_vectors,target,test_size=0.3,random_state=42)\n#X_1, X_test, y_1, y_test = cross_validation.train_test_split(X, y, test_size=0.3, random_state=0)\n\n# split the train data set into cross validation train and cross validation test\n#X_tr3, X_cv3, y_tr3, y_cv3 = cross_validation.train_test_split(X_train3, y_train3, test_size=0.3)\n\nfor i in range(1,30,2):\n    # instantiate learning model (k = 30)\n    knn = KNeighborsClassifier(n_neighbors=i)\n\n    # fitting the model on crossvalidation train\n    knn.fit(X_train3, y_train3)\n\n    # predict the response on the crossvalidation train\n    pred = knn.predict(X_cv3)\n\n    # evaluate CV accuracy\n    acc = accuracy_score(y_cv3, pred, normalize=True) * float(100)\n    print('\\nCV accuracy for k = %d is %d%%' % (i, acc))\n    \nknn = KNeighborsClassifier(1)\nknn.fit(X_train3,y_train3)\npred = knn.predict(X_test3)\nacc = accuracy_score(y_test3, pred, normalize=True) * float(100)\nprint('\\n****Test accuracy for k = 1 is %d%%' % (acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19b366f4d5ab6075c76ddde11200586715be9fc5"},"cell_type":"code","source":"\n\n# creating odd list of K for KNN\nmyList = list(range(0,50))\nneighbors = list(filter(lambda x: x % 2 != 0, myList))\n\n# empty list that will hold cv scores\ncv_scores = []\n\n# perform 10-fold cross validation\nfor k in neighbors:\n    knn = KNeighborsClassifier(n_neighbors=k)\n    scores = cross_val_score(knn, X_train3, y_train3, cv=10, scoring='accuracy')\n    cv_scores.append(scores.mean())\n\n# changing to misclassification error\nMSE = [1 - x for x in cv_scores]\n\n# determining best k\noptimal_k = neighbors[MSE.index(min(MSE))]\nprint('\\nThe optimal number of neighbors is %d.' % optimal_k)\n\n# plot misclassification error vs k \nplt.plot(neighbors, MSE)\n\nfor xy in zip(neighbors, np.round(MSE,3)):\n    plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')\n\nplt.xlabel('Number of Neighbors K')\nplt.ylabel('Misclassification Error')\nplt.show()\n\nprint(\"the misclassification error for each k value is : \", np.round(MSE,3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5f9f5c9c7bdb98b79aed55b2bcf269b906792cbc"},"cell_type":"code","source":"# ============================== KNN with k = optimal_k ===============================================\n# instantiate learning model k = optimal_k\nknn_optimal = KNeighborsClassifier(n_neighbors=optimal_k)\n\n# fitting the model\nknn_optimal.fit(X_train3, y_train3)\n\n# predict the response\npred = knn_optimal.predict(X_test3)\n\n# evaluate accuracy\nacc = accuracy_score(y_test3, pred) * 100\nprint('\\nThe accuracy of the knn classifier for k = %d is %f%%' % (optimal_k, acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8aed715ba8faed962ab497491ba17050ec7d554"},"cell_type":"code","source":"\n# TF-IDF weighted Word2Vec\ntfidf_feat = tf_idf_vect.get_feature_names() # tfidf words/col-names\n# final_tf_idf is the sparse matrix with row= sentence, col=word and cell_val = tfidf\n\ntfidf_sent_vectors = []; # the tfidf-w2v for each sentence/review is stored in this list\nrow=0;\nfor sent in list_of_sent: # for each review/sentence \n    sent_vec = np.zeros(50) # as word vectors are of zero length\n    weight_sum =0; # num of words with a valid vector in the sentence/review\n    for word in sent: # for each word in a review/sentence\n        if word in w2v_words:\n            vec = w2v_model.wv[word]\n            # obtain the tf_idfidf of a word in a sentence/review\n            tf_idf = final_tf_idf[row, tfidf_feat.index(word)]\n            sent_vec += (vec * tf_idf)\n            weight_sum += tf_idf\n    if weight_sum != 0:\n        sent_vec /= weight_sum\n    tfidf_sent_vectors.append(sent_vec)\n    row += 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c5faf98b8e0996abaa94ee565f08cc64707e558b"},"cell_type":"code","source":"# split the data set into train and test\n#X_train4,X_test4,y_train4,y_test4 = train_test_split(tfidf_sent_vectors,target,test_size=0.3,random_state=42)\n#X_1, X_test, y_1, y_test = cross_validation.train_test_split(X, y, test_size=0.3, random_state=0)\n\n# split the train data set into cross validation train and cross validation test\n#X_tr4, X_cv4, y_tr4, y_cv4 = cross_validation.train_test_split(X_train4, y_train4, test_size=0.3)\n\n\n\n\nX_train4 =  tfidf_sent_vectors[:math.ceil(len(final)*.6)] \nX_cv4 = tfidf_sent_vectors[round(len(final)*.6):round(len(final)*.8)]\nX_test4 = tfidf_sent_vectors[round(len(final)*.8):]\n\ny_train4 = y[:round(len(final)*.6)]\ny_cv4 =  y[round(len(final)*.6):round(len(final)*.8)]\ny_test4 = y[round(len(final)*.8):]\n\n\n\n\n\n\n\nfor i in range(1,30,2):\n    # instantiate learning model (k = 30)\n    knn = KNeighborsClassifier(n_neighbors=i)\n\n    # fitting the model on crossvalidation train\n    knn.fit(X_train4, y_train4)\n\n    # predict the response on the crossvalidation train\n    pred = knn.predict(X_cv4)\n\n    # evaluate CV accuracy\n    acc = accuracy_score(y_cv4, pred, normalize=True) * float(100)\n    print('\\nCV accuracy for k = %d is %d%%' % (i, acc))\n    \nknn = KNeighborsClassifier(1)\nknn.fit(X_train4,y_train4)\npred = knn.predict(X_test4)\nacc = accuracy_score(y_test4, pred, normalize=True) * float(100)\nprint('\\n****Test accuracy for k = 1 is %d%%' % (acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b9a9756a47ae839bdca83b690d3b0380c489b0bf"},"cell_type":"code","source":"\n\n# creating odd list of K for KNN\nmyList = list(range(0,50))\nneighbors = list(filter(lambda x: x % 2 != 0, myList))\n\n# empty list that will hold cv scores\ncv_scores = []\n\n# perform 10-fold cross validation\nfor k in neighbors:\n    knn = KNeighborsClassifier(n_neighbors=k)\n    scores = cross_val_score(knn, X_train4, y_train4, cv=10, scoring='accuracy')\n    cv_scores.append(scores.mean())\n\n# changing to misclassification error\nMSE = [1 - x for x in cv_scores]\n\n# determining best k\noptimal_k = neighbors[MSE.index(min(MSE))]\nprint('\\nThe optimal number of neighbors is %d.' % optimal_k)\n\n# plot misclassification error vs k \nplt.plot(neighbors, MSE)\n\nfor xy in zip(neighbors, np.round(MSE,3)):\n    plt.annotate('(%s, %s)' % xy, xy=xy, textcoords='data')\n\nplt.xlabel('Number of Neighbors K')\nplt.ylabel('Misclassification Error')\nplt.show()\n\nprint(\"the misclassification error for each k value is : \", np.round(MSE,3))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"540527d3bfd96715d87398519f5125128893ba23"},"cell_type":"code","source":"# ============================== KNN with k = optimal_k ===============================================\n# instantiate learning model k = optimal_k\nknn_optimal = KNeighborsClassifier(n_neighbors=optimal_k)\n\n# fitting the model\nknn_optimal.fit(X_train4, y_train4)\n\n# predict the response\npred = knn_optimal.predict(X_test4)\n\n# evaluate accuracy\nacc = accuracy_score(y_test4, pred) * 100\nprint('\\nThe accuracy of the knn classifier for k = %d is %f%%' % (optimal_k, acc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4df0e8e7b4286b0aaa8c0d2a3d43ec558c7fed9c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}