{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport nltk #for NLP processing and sentiment analysis\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer #for calculating sentiment scores\nfrom textblob import TextBlob #another approach of sentiment analysis\nimport plotly.plotly as py #For interactive Data Visualization\nimport plotly.graph_objs as go #For interactive Data Visualization\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.\n\ntrain_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"train_df.head()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e23727fb04c07971058368adb04105620c656070"},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9764c5c64e0d4e9cad8cf094d7753a62ee96aa01"},"cell_type":"markdown","source":"**Split The training dataset into two, one with insencere and another with sincere questions:**"},{"metadata":{"trusted":true,"_uuid":"ef857a13be0c075a0f3c5a601cb44fc1b49d26b4"},"cell_type":"code","source":"\ntrain1_df = train_df[train_df[\"target\"]==1]\ntrain0_df = train_df[train_df[\"target\"]==0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"219227fd7ad50718cfc384caa8361400c0801c7f"},"cell_type":"code","source":"train1_df.head()\nprint(\"Insincere Group shape : \", train1_df.shape)\nprint(\"Sincere Group shape : \", train0_df.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"96fec17c886adc1ab9e71cedee506e2218b96439"},"cell_type":"code","source":"train0_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b69be80c73ff7ea42bd9ae822c8031857c200a06"},"cell_type":"code","source":"train1_df.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"807b30dcd383fd597e35bdfb4b32e6b18671fe5e"},"cell_type":"markdown","source":"**Try the TextBlob sentiment analysis package first, on the insicence group:**"},{"metadata":{"trusted":true,"_uuid":"b59c61b5c0ddaf54b416ff2cce6380d6365647c9"},"cell_type":"code","source":"train1_df[['polarity','subjectivity']] = train1_df['question_text'].apply(lambda Text: pd.Series(TextBlob(Text).sentiment))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"97a891f99f1c89f5189c9855a099e946e43472ba"},"cell_type":"code","source":"print(train1_df['polarity'].mean())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8614d9b1916c7ed5b66bae958f457a4b52873c8e"},"cell_type":"code","source":"print(train1_df['subjectivity'].mean())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4bd35812540adcbbd216a3fd8f7ebfe7ae148b5e"},"cell_type":"markdown","source":"A simple Histogram of the polarity distribution, a feature engineered from the TextBlob package, on the Insincere Group, the basic graph (and the average) show that there is a tendency towards *polarity***** within the Insincere group.  The distribution turn towards (-1)."},{"metadata":{"trusted":true,"_uuid":"bab82628931402a3fe10df1145c71c318ba156d7"},"cell_type":"code","source":"hist = train1_df['polarity'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3c25e89841dbdade5f3f14b0247a676d8e228f05"},"cell_type":"code","source":"hist = train1_df['subjectivity'].hist()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"518923a9e910b79c600afc9c5188061088a6d71f"},"cell_type":"code","source":"from nltk.sentiment.vader import SentimentIntensityAnalyzer\nsid = SentimentIntensityAnalyzer()\ntrain1_df['sentiment_scores'] = train1_df['question_text'].apply(sid.polarity_scores)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"60c0824cd48cf3baf08d4b9e7cbb42b6dde7286d"},"cell_type":"code","source":"train1_df['sentiment'] = train1_df['sentiment_scores'].apply(lambda x: x['compound'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bebc3ea31b79a19d3b12eaae73eea282a6db553f"},"cell_type":"markdown","source":"When using the sentiment scores approach **(from nltk.sentiment.vader)**, we find a concentration towards the negative values, and the mean is quite low:** (-0.099)**"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"a7099dcab70283e44c21867f69722e1cdcb52c61"},"cell_type":"code","source":"hist = train1_df['sentiment'].hist()\nprint(train1_df['sentiment'].mean())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8d9194816caac68aea74a7bcb2ee867995408856"},"cell_type":"markdown","source":"*To be continued...*****"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}