{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nimport gensim\nprint(os.listdir('../input/embeddings/GoogleNews-vectors-negative300/'))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"path = \"../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\"\nembeddings = gensim.models.KeyedVectors.load_word2vec_format(path,binary = True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"733ffa6dc81dd7099316c1dd310053df02600d79"},"cell_type":"markdown","source":"# Class - count"},{"metadata":{"trusted":true,"_uuid":"bb211a28d82bf377b690c58d47e45a194551b600"},"cell_type":"code","source":"hotstar = pd.read_csv('https://bit.ly/2W21FY7')\nhotstar['Sentiment_Manual'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c91bd4adc4345d6ce49adb698800833cd9924d7"},"cell_type":"code","source":"hotstar.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae363bca2375902c8a21884cf4fb6560ed42a0e1"},"cell_type":"code","source":"target0 = hotstar[hotstar['Sentiment_Manual'] == 'Neutral']\ntarget1 = hotstar[hotstar['Sentiment_Manual'] == 'Positive']\ntarget2 = hotstar[hotstar['Sentiment_Manual'] == 'Negative']\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bb6a63402b72ad89b89252f8c428d799d0347f76"},"cell_type":"code","source":"from wordcloud import WordCloud\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5762a553c46fe855564d6afb22f5524bf432afbd"},"cell_type":"markdown","source":"## CLOUD FOR NEUTRAL"},{"metadata":{"trusted":true,"_uuid":"306b16aabf333d2f928da4e4389034b889a69385"},"cell_type":"code","source":"doc0 = target0['Reviews']\nwc0 = WordCloud(background_color='white').generate(''.join(doc0))\nplt.imshow(wc0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7138f6c4c80236d0a657d8f0a5193d319974a279"},"cell_type":"markdown","source":"# CLOULD FOR POSITIVE"},{"metadata":{"trusted":true,"_uuid":"b7975b9e0200da6a80081f55bbeea2dada79218a"},"cell_type":"code","source":"doc1 = target1['Reviews']\nwc1 = WordCloud(background_color='white').generate(''.join(doc1))\nplt.imshow(wc1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"376cad539f2387262d350c22be6bf72cd6698022"},"cell_type":"markdown","source":"# CLOUD FOR NEGATIVE"},{"metadata":{"trusted":true,"_uuid":"81cbef692deb574cc5064e3c96043e941bfa0f5c"},"cell_type":"code","source":"doc2 = target2['Reviews']\nwc2 = WordCloud(background_color='white').generate(''.join(doc2))\nplt.imshow(wc2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a98bd7e61d0a628b27f025b253da8e63c5aa7022"},"cell_type":"code","source":"import nltk","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"609f0c777a439da73800bcf590b44e82bd901990"},"cell_type":"markdown","source":"# data cleaning"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"f96e8c51956a9201c7ff8430884fb80a640ac81d"},"cell_type":"code","source":"#data cleaning\nstopwords = nltk.corpus.stopwords.words('english')\n\nstemmer = nltk.stem.PorterStemmer()\n\ndef clean_sentence(doc):\n    words = doc.split(' ')\n    words_clean = [stemmer.stem(word) for word in words if word not in stopwords]\n    doc_clean = ' '.join(words_clean)\n    return doc_clean\n\ndocs = hotstar['Reviews'].str.lower().str.replace('[^a-z ]','')\ndocs_clean = docs.apply(clean_sentence)\n\ndocs_clean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d92070fc784c70034e750f0609d9da132eb5daf9"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(docs_clean, hotstar['Sentiment_Manual'],\n                                                    test_size=0.2, \n                                                    random_state=100)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"347668c6d6d1d1b66dc2e07dc3cc2a52e70b1bfd"},"cell_type":"markdown","source":"# COUNT VECTORIZER , min df = 5"},{"metadata":{"trusted":true,"_uuid":"478ff886e15c56f3a9286f100df409b27d78e1f0"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b9750a7023f2fad345c64046ec03ea4cd6b083e"},"cell_type":"code","source":"vectorizer = CountVectorizer(min_df=5).fit(X_train)\nX_train = vectorizer.transform(X_train)\nX_test = vectorizer.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6395484031157dce3f5e1060b209bf65df8a81fe"},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nfrom sklearn.metrics import f1_score,accuracy_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e797a5c24da7eab4bc3db260413e38aad2112f63"},"cell_type":"code","source":"model_mnb = MultinomialNB().fit(X_train,y_train)\ntest_pred = model_mnb.predict(X_test)\nprint(accuracy_score(y_test,test_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e04828fcd394c53ae6e2538b9d06e955e94d2797"},"cell_type":"code","source":"from sklearn.ensemble import  RandomForestClassifier,AdaBoostClassifier\nmodel_ran = RandomForestClassifier(n_estimators = 300).fit(X_train,y_train)\ntest_pred = model_ran.predict(X_test)\nprint(accuracy_score(y_test,test_pred))\n\nprint(\"Ada\")\n\n\nmodel_ada = AdaBoostClassifier(n_estimators = 100).fit(X_train,y_train)\ntest_pred = model_ada.predict(X_test)\nprint(accuracy_score(y_test,test_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"df4c54c09a3a8ea7dfbe227fb520fe9353cd0fc9"},"cell_type":"markdown","source":"## TFIDF "},{"metadata":{"trusted":true,"_uuid":"ffd7e5540976666e814a9dea2d0744df4bd04bb8"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(docs_clean, hotstar['Sentiment_Manual'],\n                                                    test_size=0.2, \n                                                    random_state=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e62f42aefc0e9fea1367aabec105de7b3485b1e8"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\ntfidf = TfidfVectorizer().fit(X_train) #it supresses the weights \nX_train = tfidf.transform(X_train)\nX_test = tfidf.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9596d178f62aedd749225a1110b389342144d86c"},"cell_type":"code","source":"model_mnb = MultinomialNB().fit(X_train,y_train)\ntest_pred = model_mnb.predict(X_test)\nprint(accuracy_score(y_test,test_pred))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2d9f9e1fdba9dca1558714aded23ce9dd137fe39"},"cell_type":"markdown","source":"## WORD VEC"},{"metadata":{"trusted":true,"_uuid":"c6330a484ac8b9f54f872eff6ae6416421c70660"},"cell_type":"code","source":"docs_vectors = pd.DataFrame()\nfor doc in docs_clean:\n    words = nltk.word_tokenize(doc)\n    temp = pd.DataFrame()\n    for word in words : \n        try: \n            word_vec = embeddings[word]\n            temp = temp.append(pd.Series(word_vec), ignore_index= True)\n        except:\n            pass #goes to the next word \n    docs_vectors = docs_vectors.append(temp.mean(), ignore_index = True)\ndocs_vectors.shape\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a9b4bf59dd48dec49411246aef748c6e7e852a4a"},"cell_type":"code","source":"s = pd.DataFrame(pd.isnull(docs_vectors).sum(axis=1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"905e132b43e302577241f54d2d04a8a5284be5d9"},"cell_type":"code","source":"s.head()\ns.columns = ['sum']\nb = s[s['sum']==300] \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5c8c196cf5c6d934c2449218f709e16b4c8fe77"},"cell_type":"code","source":"X = docs_vectors.drop(b.index)\ny = hotstar['Sentiment_Manual'].drop(b.index)\n\nX.shape, y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"433e3950ca5a8e6c770b301de9dec86a86f06163"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_x , test_x , train_y, test_y = train_test_split(X,y,\n                                                     test_size = 0.2 , random_state = 100 )\ntrain_x.shape, test_x.shape, train_y.shape, test_y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78de17e836cdad421d502aa57672786d2876e945"},"cell_type":"code","source":"from sklearn.ensemble import  RandomForestClassifier,AdaBoostClassifier\nmodel_ran_wv = RandomForestClassifier(n_estimators = 300).fit(train_x,train_y)\ntest_pred_wv = model_ran_wv.predict(test_x)\nprint(accuracy_score(test_y,test_pred_wv))\n\nprint(\"Ada\")\n\n\nmodel_ada_wv = AdaBoostClassifier(n_estimators = 100).fit(train_x,train_y)\ntest_pred_ada= model_ada_wv.predict(test_x)\nprint(accuracy_score(test_y,test_pred_ada))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b847efaef20dcd4c869728ba06b2e1054cff1995"},"cell_type":"markdown","source":"# VADER\n"},{"metadata":{"trusted":true,"_uuid":"a843c9e39b0ce0fd5846ae3241e12e2e0a6b8e36"},"cell_type":"code","source":"from nltk.sentiment import SentimentIntensityAnalyzer\n\nanalyzer = SentimentIntensityAnalyzer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5fb7765cf5da56efdebd49bdb9c65c21dc947c13"},"cell_type":"code","source":"def get_sentiment(sentence , analyzer = analyzer):\n    compound = analyzer.polarity_scores(sentence)['compound']\n    if compound > 0.1 :\n        return 'Positive'\n    if compound < 0.1 : \n        return 'Negative'\n    else: \n        return 'Neutral'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2c0de052f34859071462f476fb17eb5f8e4dc5b5"},"cell_type":"code","source":"hotstar['sentiment_vader'] = hotstar['Reviews'].apply(get_sentiment)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ae42e3aabd95d4bcfd20dd28d41b87596d425bf"},"cell_type":"code","source":"accuracy_score(hotstar['Sentiment_Manual'],hotstar['sentiment_vader'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bda8b5b87286d7fecdeddcf29f82a5c7342f1cd9"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}