{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Problem Statement\n\n1. The json file contains the **stocktwit data**, the **timestamp** of collecting the tweet and the ticker(stock tdentifier).\n2. The tagged **sentiment ranges from 0-3**.\n\nBuild a model to predict the sentiment of the stocktwit\n* The json files have a structure as follows:\n\n* { ‘records’: [\n\n* {\n\n* 'stocktwit_tweet': ‘$TSLA is a definite buy today’,\n\n* 'sentiment_score': ‘3’,\n\n* 'timestamp': ‘2018-07-01 00:00:09+00:00’,\n\n* 'ticker': ‘TSLA’\n\n* },\n\n* {..},\n\n* {..}\n\n* ]\n\n* }","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Import the libraries\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Do not display warnings in notebook \nimport warnings\nwarnings.filterwarnings('ignore')\n\npd.set_option('display.max_columns', 500)\npd.options.display.max_seq_items = 2000\n\nimport sys\n\nnp.set_printoptions(threshold=sys.maxsize)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing the json file\nJ_tweets = pd.read_json (\"../input/financial-data/train_data_JSON.json\")\n# df = pd.read_json(filename)\n# df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Importing the json file\nJSON_tweets_test = pd.read_json(\"../input/financial-data/test_data.json\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print the shape of the tweeets data\nJ_tweets.records.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"JSON_tweets_test.records.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print the first json record\nJ_tweets.records[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"JSON_tweets_test.records[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert the json data into dataframe\nfrom pandas.io.json import json_normalize","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets_data = json_normalize(J_tweets.records)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets_data1 = json_normalize(JSON_tweets_test.records)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Save the dataframe into a csv file\ntweets_data.to_csv(\"tweets.csv\",index=False)\ntweets_data1.to_csv(\"tweets1.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Exploring the data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import the tweets csv file\ntweets = pd.read_csv(\"tweets.csv\")\ntweets1 = pd.read_csv(\"tweets.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.skew(), tweets.kurt()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.sentiment_score.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Sentiment_count = tweets['sentiment_score'].value_counts()\nplt.figure(figsize=(10,4))\nsns.barplot(Sentiment_count.index, Sentiment_count.values, alpha=0.8,)\nplt.ylabel(\"COUNT\")\nplt.xlabel(\"sentiment_score\")\nplt.title('sentiment_score counts across the text data', loc='Center', fontsize=19)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets1.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of words in train data\ntweets['word_count'] = tweets['stocktwit_tweet'].apply(lambda x: len(str(x).split(\" \")))\ntweets[['stocktwit_tweet','word_count']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import cufflinks as cf\ncf.go_offline()\ncf.set_config_file(offline=False, world_readable=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of words in testdata\ntweets1['word_count'] = tweets1['stocktwit_tweet'].apply(lambda x: len(str(x).split(\" \")))\ntweets1[['stocktwit_tweet','word_count']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets['word_count'].iplot(\n    kind='hist',\n    bins=100,\n    xTitle='word count',\n    linecolor='black',\n    yTitle='count',\n    title='Review Text Word Count Distribution')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Avg word length\ndef avg_word(sentence):\n    words = sentence.split()\n    return (sum(len(word) for word in words)/len(words))\n\ntweets['avg_word'] = tweets['stocktwit_tweet'].apply(lambda x: avg_word(x))\ntweets[['stocktwit_tweet','avg_word']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets['avg_word'].iplot(\n    kind='hist',\n    bins=100,\n    xTitle='word count',\n    linecolor='black',\n    yTitle='count',\n    title='Review Text Word Count Distribution')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Avg word length in test data\ndef avg_word(sentence):\n    words = sentence.split()\n    return (sum(len(word) for word in words)/len(words))\n\ntweets1['avg_word'] = tweets1['stocktwit_tweet'].apply(lambda x: avg_word(x))\ntweets1[['stocktwit_tweet','avg_word']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets1['avg_word'].iplot(\n    kind='hist',\n    bins=100,\n    xTitle='word count',\n    linecolor='black',\n    yTitle='count',\n    title='Review Text Word Count Distribution')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of special characters in train data\ntweets['hastags'] = tweets['stocktwit_tweet'].apply(lambda x: len([x for x in x.split() if x.startswith('#')]))\ntweets[['stocktwit_tweet','hastags']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of special characters in test data\ntweets1['hastags'] = tweets1['stocktwit_tweet'].apply(lambda x: len([x for x in x.split() if x.startswith('#')]))\ntweets1[['stocktwit_tweet','hastags']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of numerics in train data\ntweets['numerics'] = tweets['stocktwit_tweet'].apply(lambda x: len([x for x in x.split() if x.isdigit()]))\ntweets[['stocktwit_tweet','numerics']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of numerics in test data\ntweets1['numerics'] = tweets1['stocktwit_tweet'].apply(lambda x: len([x for x in x.split() if x.isdigit()]))\ntweets1[['stocktwit_tweet','numerics']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of uppercase words in train data\ntweets['upper'] = tweets['stocktwit_tweet'].apply(lambda x: len([x for x in x.split() if x.isupper()]))\ntweets[['stocktwit_tweet','upper']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of uppercase words in test data\ntweets1['upper'] = tweets1['stocktwit_tweet'].apply(lambda x: len([x for x in x.split() if x.isupper()]))\ntweets1[['stocktwit_tweet','upper']].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Timestamp","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Conversion in train_data\ntweets['Dates'] = pd.to_datetime(tweets['timestamp']).dt.date\ntweets['Time'] = pd.to_datetime(tweets['timestamp']).dt.time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Conversion in test_data\ntweets1['Dates'] = pd.to_datetime(tweets1['timestamp']).dt.date\ntweets1['Time'] = pd.to_datetime(tweets1['timestamp']).dt.time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"days = {0:'Mon',1:'Tues',2:'Weds',3:'Thurs',4:'Fri',5:'Sat',6:'Sun'}\n\ntweets[\"Dateoftweet\"]=pd.to_datetime(tweets[\"Dates\"])\ntweets[\"Day\"]=tweets[\"Dateoftweet\"].dt.day\ntweets[\"month\"]=tweets[\"Dateoftweet\"].dt.month\ntweets[\"year\"]=tweets[\"Dateoftweet\"].dt.month\ntweets[\"dayOftheweek\"]=tweets[\"Dateoftweet\"].dt.dayofweek\n\ntweets['dayOftheweek'] = tweets['dayOftheweek'].apply(lambda x: days[x])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.drop(['timestamp'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.dtypes","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Tweet_Day_Count = tweets['Day'].value_counts()\nplt.figure(figsize=(10,4))\nsns.barplot(Tweet_Day_Count.index, Tweet_Day_Count.values, alpha=0.8)\nplt.ylabel(\"Number Of Tweet\")\nplt.xlabel(\"Tweets By Days\")\nplt.title('Total tweets count by Day', loc='Center', fontsize=14)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Stopwords","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of stop words in train_data\nfrom nltk.corpus import stopwords\nstop = stopwords.words('english')\n\ntweets['stopwords'] = tweets['stocktwit_tweet'].apply(lambda x: len([x for x in x.split() if x in stop]))\ntweets[['stocktwit_tweet','stopwords']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Number of characters\ntweets1['char_count'] = tweets1['stocktwit_tweet'].str.len() ## this also includes spaces\ntweets1[['stocktwit_tweet','char_count']].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print the shape of the dataframe(train_data)\ntweets.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print the shape of the dataframe(test_data)\ntweets1.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Sentiment_count = tweets['sentiment_score'].value_counts()\nplt.figure(figsize=(10,4))\nsns.barplot(Sentiment_count.index, Sentiment_count.values, alpha=0.8)\nplt.ylabel(\"Count\")\nplt.xlabel(\"sentiment Count\")\nplt.title('sentiment counts across the tweet data', loc='Center', fontsize=14)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets1.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Checking the null values in the data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets1.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### find the sentiment score for train_data\ntweets.sentiment_score.value_counts()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ticker","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preprocess the tweets data of ticker feature (train_data)\ntweets['ticker']=tweets['ticker'].apply(lambda x:x.lower().replace('$',''))\ntweets['ticker'] = tweets['ticker'].apply(lambda x: '$'+x)\nlen(tweets['ticker'].str.lower().unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preprocess the tweets data of ticker feature(test_data)\ntweets1['ticker']=tweets1['ticker'].apply(lambda x:x.lower().replace('$',''))\ntweets1['ticker'] = tweets1['ticker'].apply(lambda x: '$'+x)\nlen(tweets1['ticker'].str.lower().unique())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"top_ticker = tweets.ticker.value_counts()[:10]\ntop_ticker","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(15,12))\nsns.barplot(top_ticker.index, top_ticker.values, alpha=0.8)\nplt.ylabel(\"Count\")\nplt.xlabel(\"Frequent words\")\nplt.title('frequency of Ticker', loc='Center', fontsize=14)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the graph between the sentiment_score and the count(train_data)\nSentiment_count = tweets['sentiment_score'].value_counts()\nplt.figure(figsize=(12,8))\nsns.barplot(Sentiment_count.index, Sentiment_count.values, alpha=0.8)\nplt.ylabel(\"COUNT\")\nplt.xlabel(\"sentiment_score\")\nplt.title('sentiment_score counts across the text data', loc='Center', fontsize=19)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import the libraries\nimport nltk\nimport re\nimport pandas as pd\n\nfrom sklearn.model_selection import train_test_split\nfrom nltk.stem import PorterStemmer\nfrom nltk.stem import WordNetLemmatizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets1.head(5)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Remove non letters","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Removing non_letters(train_data)\ntweets[\"stocktwit_tweet\"]=tweets[\"stocktwit_tweet\"].apply(lambda x:re.sub(\"[^A-Za-z]\", \" \", x.strip()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Removing non_letters-(test_data) \ntweets1[\"stocktwit_tweet\"]=tweets1[\"stocktwit_tweet\"].apply(lambda x:re.sub(\"[^A-Za-z]\", \" \", x.strip()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Convert into Lower Case","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"# Converting into Lower Cases- (train_data)\ntweets['stocktwit_tweet'] = tweets['stocktwit_tweet'].apply(lambda x: \" \".join(x.lower() for x in x.split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Converting into Lower Cases- (test_data)\ntweets1['stocktwit_tweet'] = tweets1['stocktwit_tweet'].apply(lambda x: \" \".join(x.lower() for x in x.split()))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Removing Numbers for dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Removing Numbers-train_data\ntweets['stocktwit_tweet'] = tweets['stocktwit_tweet'].str.replace('[\\d]', '')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Removing Numbers- (test_data)\ntweets1['stocktwit_tweet'] = tweets1['stocktwit_tweet'].str.replace('[\\d]', '')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Removing Punctuational marks-()","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Removing Punctuational marks- (train_data)\ntweets['stocktwit_tweet'] = tweets['stocktwit_tweet'].str.replace('[^\\w\\s]','')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Removing Punctuational marks- (test_data)\ntweets1['stocktwit_tweet'] = tweets1['stocktwit_tweet'].str.replace('[^\\w\\s]','')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Removing Stop words","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Removing Stop words- (train_data)\nstop = stopwords.words('english')\ntweets['stocktwit_tweet'] = tweets['stocktwit_tweet'].apply(lambda x: \" \".join(x for x in x.split() if x not in stop))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Common word removal (train_data)\nfreq = pd.Series(' '.join(tweets['stocktwit_tweet']).split()).value_counts()[:10]\nprint(freq)\n\n# Remove these words as their presence will not of any use in classification of our text data.\nfreq = list()\ntweets['stocktwit_tweet'] = tweets['stocktwit_tweet'].apply(lambda x: \" \".join(x for x in x.split() if x not in freq))\ntweets['stocktwit_tweet'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Common word removal(test_data)\nfreq = pd.Series(' '.join(tweets1['stocktwit_tweet']).split()).value_counts()[:10]\nprint(freq)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Remove these words as their presence will not of any use in classification of our text data.\nfreq = list()\ntweets1['stocktwit_tweet'] = tweets1['stocktwit_tweet'].apply(lambda x: \" \".join(x for x in x.split() if x not in freq))\ntweets1['stocktwit_tweet'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Rare words removal (train_data)\nfreq = pd.Series(' '.join(tweets['stocktwit_tweet']).split()).value_counts()[-10:]\nprint(freq)\n\nfreq = list(freq.index)\ntweets['stocktwit_tweet'] = tweets['stocktwit_tweet'].apply(lambda x: \" \".join(x for x in x.split() if x not in freq))\ntweets['stocktwit_tweet'].head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Rare words removal (test_data)\nfreq = pd.Series(' '.join(tweets1['stocktwit_tweet']).split()).value_counts()[-10:]\nprint(freq)\n\nfreq = list(freq.index)\ntweets1['stocktwit_tweet'] = tweets1['stocktwit_tweet'].apply(lambda x: \" \".join(x for x in x.split() if x not in freq))\ntweets1['stocktwit_tweet'].head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Word Frequency","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Word Frequency- train_data\nWord_freq = pd.Series(' '.join(tweets['stocktwit_tweet']).split()).value_counts()[:10]\nWord_freq","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Word Frequency- test_data\nWord_freq = pd.Series(' '.join(tweets1['stocktwit_tweet']).split()).value_counts()[:10]\nWord_freq","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Word Frequency Plot - \nplt.figure(figsize=(10,4))\nsns.barplot(Word_freq.index, Word_freq.values, alpha=0.8)\nplt.ylabel(\"Count\")\nplt.xlabel(\"Frequent words\")\nplt.title('Frequent words in review_data', loc='Center', fontsize=14)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Word Frequency Plot - (test_data)\nplt.figure(figsize=(10,4))\nsns.barplot(Word_freq.index, Word_freq.values, alpha=0.8)\nplt.ylabel(\"Count\")\nplt.xlabel(\"Frequent words\")\nplt.title('Frequent words in review_data', loc='Center', fontsize=14)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Stemming (train_data)\nst = PorterStemmer()\ntweets[\"stocktwit_tweet\"] = tweets[\"stocktwit_tweet\"].apply(lambda x: \" \".join([st.stem(word)\n                                                                   for word in x.split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Stemming (test_data)\nst = PorterStemmer()\ntweets1[\"stocktwit_tweet\"] = tweets1[\"stocktwit_tweet\"].apply(lambda x: \" \".join([st.stem(word)\n                                                                   for word in x.split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":" nltk.download('wordnet')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Lemmatization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Lemmatization (train_data)\nLem = WordNetLemmatizer()\ntweets[\"stocktwit_tweet\"] = tweets[\"stocktwit_tweet\"].apply(lambda x: \" \".join([Lem.lemmatize(word)\n                                                          for word in x.split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# # Lemmatization (test_data)\nLem = WordNetLemmatizer()\ntweets1[\"stocktwit_tweet\"] = tweets1[\"stocktwit_tweet\"].apply(lambda x: \" \".join([Lem.lemmatize(word)\n                                                           for word in x.split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets1.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Splitting into train and test\nX_train,X_test,Y_train,Y_test = train_test_split(tweets['stocktwit_tweet'],\n                                                 tweets['sentiment_score'],\n                                                 test_size=0.25,\n                                                 random_state=7)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n# Print the shape of train and test\nprint(X_train.shape)\nprint(Y_train.shape)\nprint(X_test.shape)\nprint(Y_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Function for cleaning the text\ndef clean_text(text):\n    text = text.lower()\n    text = re.sub(r'@[a-zA-Z0-9_]+', '', text)   \n    text = re.sub(r'https?://[A-Za-z0-9./]+', '', text)   \n    text = re.sub(r'www.[^ ]+', '', text)  \n    text = re.sub(r'[a-zA-Z0-9]*www[a-zA-Z0-9]*com[a-zA-Z0-9]*', '', text)  \n    text = re.sub(r'[^a-zA-Z]', ' ', text)   \n    text = [token for token in text.split() if len(token) > 2]\n    text = ' '.join(text)\n    #text = emoji.demojize(text)\n    \n    return text\n\nX_train = X_train.apply(clean_text)\nX_test =X_test.apply(clean_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\ntfidf_vect = TfidfVectorizer(analyzer='word', stop_words=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b', ngram_range=(1, 1), max_df=1.0, \n                             min_df=3, max_features=None, binary=False, norm='l2', use_idf=True, smooth_idf=True, sublinear_tf=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit the model\nX_train_tfidf = tfidf_vect.fit_transform(X_train)\nX_test_tfidf = tfidf_vect.transform(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print the shape of the train and test\nprint(X_train_tfidf.shape)\nprint(X_test_tfidf.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Build a naive_bayes model\nfrom sklearn.metrics import f1_score, accuracy_score,confusion_matrix\nfrom sklearn.metrics import classification_report\nfrom sklearn.naive_bayes import MultinomialNB\nnb_clf = MultinomialNB().fit(X_train_tfidf, Y_train)\npred_test = nb_clf.predict(X_test_tfidf)\n\n# Print the mertics\nprint('f1_score       :', f1_score(Y_test, pred_test, average='macro'))\nprint('accuracy score :', accuracy_score(Y_test, pred_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print the classification reports\nprint(classification_report(Y_test, pred_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Import the libraries\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.linear_model import SGDClassifier\n\n# Build the model\nsgd = SGDClassifier(loss='log', max_iter=200, random_state=0, class_weight='balanced')\novr = OneVsRestClassifier(sgd)\novr.fit(X_train_tfidf, Y_train)\ny_pred_class = ovr.predict(X_test_tfidf)\n\n# Print the metrics score\nprint('f1_score       :', f1_score(Y_test, y_pred_class, average='macro'))\nprint('accuracy score :', accuracy_score(Y_test, y_pred_class))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print the classification report\nprint(classification_report(Y_test, y_pred_class)) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Print the confusion matrix\nconfusion_matrix(Y_test,y_pred_class)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Validation DataFrame\nvalidation = pd.DataFrame({'stocktwit_tweet':X_test,\n                           'predicted_sentiment':''})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation['predicted_sentiment']=y_pred_class\nvalidation['actual_sentiment']=Y_test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Comparision of the predicted and actual sentiment of test data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"validation.head(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation['predicted_sentiment'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nvalidation['actual_sentiment'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Preprocessing the test text\nX_test_data = tweets1['stocktwit_tweet'].apply(clean_text)\nX_test_data_tfidf = tfidf_vect.transform(X_test_data )\nprint(X_test_data_tfidf.shape)\n\ny_pred_class_data = ovr.predict(X_test_data_tfidf)\ntweets1['sentiment_score'] = y_pred_class_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tweets1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}