{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport re  # for removing unnecessary tags\nimport nltk\nfrom bs4 import BeautifulSoup # for removing the HTML tags\nfrom nltk.corpus import stopwords # for removing stopwords\nfrom nltk.tokenize import word_tokenize\n\npd.set_option(\"display.max_colwidth\", 500)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2fda6eb15ebb0f1753e439f5d37c86e08bb35aa3"},"cell_type":"code","source":"train = pd.read_csv(\"../input/labeledTrainData.tsv\", header = 0, delimiter = '\\t')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83cef4adc4bad75363a23761e799e45db844c88f"},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30f777d156553857a243f1143dcaf4d1e6b742f9"},"cell_type":"code","source":"train.sentiment.value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ebe5cf07fc2b9680aa3244583dbb80945cd46f0"},"cell_type":"markdown","source":"Clearly the dataset is balanced,i.e equal number of positive and negative reviews"},{"metadata":{"trusted":true,"_uuid":"4089738c9214c9b1c60442269de2f423c945556a"},"cell_type":"code","source":"##cleaning of data for sentiment analysis\n\n# Removing Stop words from Reviews\nstop_words = set(stopwords.words('english'))\n\n# LEMMATIZING\nfrom nltk import WordNetLemmatizer\nlemma = WordNetLemmatizer()\n\n\ndef cleaning(line):\n    \n    #Removing unecessary html tags\n    soup = BeautifulSoup(line,\"lxml\")\n    line = soup.get_text()\n    \n    # Removing punctuations and special characters\n    line = re.sub(r\"[^\\w\\s]\",\"\",line)\n    \n    #tokenizing of the review\n    tokens = word_tokenize(line)\n    \n    #eliminating the stop words\n    tokens = [x for x in tokens if x not in stop_words]\n    \n    #lemmatizing the tokens\n    tokens = [lemma.lemmatize(x, pos = 'v') for x in tokens]\n    \n    # filter out short tokens\n    tokens = [word for word in tokens if len(word) > 1]\n    line = ' '.join(tokens)\n    return line\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5835d142fef43a152488d386a5f495095c911f60"},"cell_type":"code","source":"train[\"review_new\"]=train.review.map(cleaning)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}