{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd \nimport os\nimport gensim\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings/GoogleNews-vectors-negative300/\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"path = \"../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\"\nembeddings = gensim.models.KeyedVectors.load_word2vec_format(path, binary=True)\n## Collection of all these word vectorings is embeddings","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9bfc8d07996d2c1115871c48ccab179d49e169d5"},"cell_type":"markdown","source":"### Reading hotstar data"},{"metadata":{"trusted":true,"_uuid":"1bea840be285cfb36f8c31ae0a2e70ba62f648c5"},"cell_type":"code","source":"hotstar = pd.read_csv('https://bit.ly/2W21FY7')\nhotstar.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f63b0f27724597f4af6a97452c00e721f1335e15"},"cell_type":"code","source":"import nltk","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a62d34d8c61b019b7465a625a81113dff71ac507"},"cell_type":"code","source":"hotstar['Sentiment_Manual'].value_counts() # Checking the count","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"94b92948a8bbdf99e64af87c041e666ceed358aa"},"cell_type":"code","source":"hotstar.isnull().sum() # Check for null values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef16f89a550d55f70c7b9066d7b51a2876cba209"},"cell_type":"markdown","source":"### Word cloud"},{"metadata":{"trusted":true,"_uuid":"9f773b5861b4874eb73e043089c37ee638d7f060"},"cell_type":"code","source":"!pip install wordcloud","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c6d9960f4e664ca350a21c10ce9674994b6bbd03"},"cell_type":"code","source":"from wordcloud import WordCloud\nimport matplotlib.pyplot as plt\n%matplotlib inline\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a1761229a75ae0b4ccd330481d1edc10a3ae96a"},"cell_type":"code","source":"Neutral = hotstar[hotstar['Sentiment_Manual'] == 'Neutral']\nPositive = hotstar[hotstar['Sentiment_Manual'] =='Positive']\nNegative = hotstar[hotstar['Sentiment_Manual' ]=='Negative']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d716fff29bfb0e96c281d686fe8053cfd1841fd"},"cell_type":"code","source":"docs0=Neutral['Lower_Case_Reviews']\nprint(len(docs0))\ndocs1=Positive['Lower_Case_Reviews']\nprint(len(docs1))\ndocs2=Negative['Lower_Case_Reviews']\nprint(len(docs2))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca0294206c5ea87a5db08e0a8f93497418ef46bf"},"cell_type":"code","source":"stopwords=nltk.corpus.stopwords.words('english')\nwc0 = WordCloud(background_color='white',stopwords=stopwords).generate(' '.join(docs0))\nplt.imshow(wc0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e383d107f5bc3760b3af695ea56c75dc8b18406b"},"cell_type":"code","source":"stopwords=nltk.corpus.stopwords.words('english')\nwc1 = WordCloud(background_color='white',stopwords=stopwords).generate(' '.join(docs1))\nplt.imshow(wc1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"689a6558b29cba2f21739b535bee5ea84f6dcc43"},"cell_type":"code","source":"stopwords=nltk.corpus.stopwords.words('english')\nwc2 = WordCloud(background_color='white',stopwords=stopwords).generate(' '.join(docs2))\nplt.imshow(wc2)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"037f01dbd139db812823d43596208223d8251c44"},"cell_type":"markdown","source":"### Data Cleaning"},{"metadata":{"trusted":true,"_uuid":"38f97cc660348d5559afec0472c150d6a2dab903"},"cell_type":"code","source":"docs=hotstar['Lower_Case_Reviews']\nlen(hotstar)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eb632ee788a9f3b560284d4a048b9aedca1af99b"},"cell_type":"code","source":"docs=docs.str.replace('[^a-z A-Z #@]', '')\ndocs.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"77fbb3a770841c3f53c820b1b94d484a25bef342"},"cell_type":"markdown","source":"### Training and Validation"},{"metadata":{"trusted":true,"_uuid":"2390f45c344b674f08dd63055711e41712f736d4"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_x,test_x,train_y,test_y=train_test_split(docs,\n                                               hotstar['Sentiment_Manual'],\n                                               test_size=0.2,\n                                               random_state=100)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"46666f6c486950c9ec4ef8a540294fd1287f6a09"},"cell_type":"markdown","source":"### Count Vectorizer"},{"metadata":{"trusted":true,"_uuid":"5d4fd0abd7bada6555bf2a35968d599e938146d9"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\nvectorizer = CountVectorizer(min_df=5).fit(train_x)\ntrain_x = vectorizer.transform(train_x)\ntest_x = vectorizer.transform(test_x)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b2babe71777d0ed0878e1cbb9472a1117861f0a4"},"cell_type":"markdown","source":"### Model Building using Count Vectorizer"},{"metadata":{"_uuid":"03d7f003d92f5415a06c195771ac673cdfa6816d"},"cell_type":"markdown","source":"### Multinomial Naive Bayes"},{"metadata":{"trusted":true,"_uuid":"e3f5f7085aa592bec7c17ae6b826a8b8b362595a"},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nfrom sklearn.metrics import accuracy_score\nmodel_mnb = MultinomialNB().fit(train_x , train_y)\ntest_pred_mnb = model_mnb.predict(test_x)\nprint(accuracy_score(test_y , test_pred_mnb))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0f9c1e533a3c84ae6afa8cdcae07cd2670161194"},"cell_type":"markdown","source":"### Random Forest"},{"metadata":{"trusted":true,"_uuid":"c36da79850ec70fb075dd9bc6502ef0ca4e72cc2"},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\nmodel = RandomForestClassifier(n_estimators=500).fit(train_x, train_y)\ntest_pred = model.predict(test_x)\naccuracy_score(test_y, test_pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"990074d86965d9762ce2db3805906753a15875be"},"cell_type":"markdown","source":"### Ada Boost"},{"metadata":{"trusted":true,"_uuid":"7d1eaf93090e42018cb22854c633486e69494275"},"cell_type":"code","source":"model = AdaBoostClassifier(n_estimators=500).fit(train_x, train_y)\ntest_pred = model.predict(test_x)\naccuracy_score(test_y, test_pred)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4752ce12abd539b6d8e131a6485bc3c26e30ea5d"},"cell_type":"markdown","source":"### TF-IDF Vectorizer"},{"metadata":{"trusted":true,"_uuid":"0c18888e6fc2050c92fef1e4edde475d47a633da"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\n\ntrain_X,test_X,train_Y,test_Y=train_test_split(docs,\n                                               hotstar['Sentiment_Manual'],\n                                               test_size=0.2,\n                                               random_state=100)\ntfid = TfidfVectorizer(min_df=2).fit(train_X)\ntrain_X = tfid.transform(train_X)\ntest_X = tfid.transform(test_X)\n\nfeatures = tfid.get_feature_names()\ntrain_X = pd.DataFrame(train_X.toarray(), columns=features)\ntest_X = pd.DataFrame(test_X.toarray(), columns=features)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"943a9666f2cfcd147bd4f1068110a046a484d37a"},"cell_type":"markdown","source":"### Multinomial Naive Bayes using TF-IDF Vectorizer"},{"metadata":{"trusted":true,"_uuid":"72d17da577bff49dd9955a01c6601e58bd11be6a"},"cell_type":"code","source":"model_tfid_mnb = MultinomialNB().fit(train_X,train_Y)\ntest_pred_tfid_mnb = model_tfid_mnb.predict(test_X)\nprint(accuracy_score(test_Y,test_pred_tfid_mnb))\n# mnb cannot take negative values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe5c39ca8f42a5cf18a26259c37a5dd1559beb0b"},"cell_type":"markdown","source":"### Word2Vec"},{"metadata":{"trusted":true,"_uuid":"33a2bc31bfa01bb331e6b364d610eb93d23cacc5"},"cell_type":"code","source":"docs_vectors = pd.DataFrame()\nfor doc in docs:\n    words = nltk.word_tokenize(doc)\n    temp = pd.DataFrame()\n    for word in words:\n        try:\n            word_vec = embeddings[word]\n            temp = temp.append(pd.Series(word_vec), ignore_index=True)\n        except:\n            pass\n    docs_vectors = docs_vectors.append(temp.mean(), ignore_index = True)\ndocs_vectors.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2487ddbea477056729e326caafd1d1f71ae6ac1b"},"cell_type":"code","source":"# Check for null values\nnull_values=pd.DataFrame(pd.isnull(docs_vectors).sum(axis = 1).sort_values(ascending = False))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c7ba5cf23927c457a08b5e39d2c6140d3cf7e14"},"cell_type":"code","source":"null_values.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b0e8eb8c5479500d9889c5065583fbf3d6df94a"},"cell_type":"code","source":"null_list = null_values.index[null_values[0]==300].tolist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"23998b8785c06b73be4046e2c8d281728b4d6f3a"},"cell_type":"code","source":"len(null_list) # Checking the length of null values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78ff43241b81d273960731c98dfa536c50669eed"},"cell_type":"code","source":"X = docs_vectors.drop(null_list)\ny = hotstar['Sentiment_Manual'].drop(null_list)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"71d00629b6776c1f00261326aca4bc1dc255eea2"},"cell_type":"markdown","source":"### Train_Test_Split"},{"metadata":{"trusted":true,"_uuid":"51a25d90ef497e4adc0cc68ecd2de5ca86d2852f"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_x,test_x,train_y,test_y = train_test_split(X,y, test_size = 0.2, random_state = 100)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"389996f1565fe0e2f0a4701f1806885397c27fbe"},"cell_type":"markdown","source":"### Model Building using Word2Vec"},{"metadata":{"_uuid":"0ddd213e7bf8d615fe57b9e71a6821c75bf0b319"},"cell_type":"markdown","source":"### Random Forest"},{"metadata":{"trusted":true,"_uuid":"74adf7288beaff47c5ed9dbef006dc419332e6be"},"cell_type":"code","source":"model_rf = RandomForestClassifier(n_estimators=100).fit(train_x, train_y)\ntest_pred_rf = model_rf.predict(test_x)\naccuracy_score(test_y, test_pred_rf)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf23ae36651bec2aa9f432304841e0d30c5c2e6b"},"cell_type":"markdown","source":"### Ada Boost"},{"metadata":{"trusted":true,"_uuid":"bb3f6082b0eef6b0c3bec6a7228c647b247139c0"},"cell_type":"code","source":"model_ab = AdaBoostClassifier(n_estimators=100).fit(train_x, train_y)\ntest_pred_ab = model_ab.predict(test_x)\naccuracy_score(test_y, test_pred_ab)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ea00fb2f723e727c5b1b2136894abf80cf956d07"},"cell_type":"markdown","source":"### Sentiment Prediction using VADER"},{"metadata":{"trusted":true,"_uuid":"0000697e1ce0cdece2ee95a36c79abc59d310da5"},"cell_type":"code","source":"from nltk.sentiment import SentimentIntensityAnalyzer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"77d88c03e42d1babc9a48020d8ce3e0fc1d1e149"},"cell_type":"code","source":"analyzer = SentimentIntensityAnalyzer()\ndef get_sentiment(sentence , analyzer = analyzer):\n    compound = analyzer.polarity_scores(sentence)['compound']\n    if compound > 0.1 :\n        return 'Positive'\n    if compound < 0.1 : \n        return 'Negative'\n    else: \n        return 'Neutral'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ea749f1d58d4b03a72c2c2f9d314b95ee756aa51"},"cell_type":"code","source":"get_sentiment(hotstar.loc[1, 'Lower_Case_Reviews'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6abb2ccde39c5d24f712f7f6421bf5d2847a5aa3"},"cell_type":"code","source":"get_sentiment(hotstar.loc[6, 'Lower_Case_Reviews'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1656fce7d688631702ec7753dba8efe58e004111"},"cell_type":"code","source":"hotstar['sentiment_vader'] = hotstar['Lower_Case_Reviews'].apply(get_sentiment)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c82ab4018fe393c84c47e798e59c448fc48b6db6"},"cell_type":"code","source":"accuracy_score(hotstar['Sentiment_Manual'],hotstar['sentiment_vader'])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}