{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\ndata=pd.read_csv(\"../input/train.csv\")\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"334c9d53810bc59a94d4c6a82ea5a9ac94c634ee"},"cell_type":"code","source":"data.iloc[22] ['question_text'] #iloc is information in that location","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b755feaa8df551a9d49b108e96add21d67ec006a"},"cell_type":"code","source":"import nltk","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dc896c230d190dbd921fac847185ee7abda3228a"},"cell_type":"code","source":"nltk.download('stopwords')\nnltk.download('punkt')\nnltk.download('vader_lexicon')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad54ce53ce3b1444d54cae82d265808836e8fecc"},"cell_type":"code","source":"!pip install wordcloud","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"445f2e301dbd9a387fb4a95881b9c0f09c67ed65"},"cell_type":"code","source":"data['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f6c97e93b3ea6f5e8247c43b82876988f69ad2cf"},"cell_type":"code","source":"#BAG OF WORD ANALYSIS","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c2dc4703a63d490bacb5eb31ac6f04f591b5e716"},"cell_type":"code","source":"from wordcloud import WordCloud\nimport matplotlib.pyplot as plt\n\nwc=WordCloud(background_color='white').generate('i love india')\nplt.imshow(wc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e53ec442eafe7486a8bd3f717785faa44769d8a5"},"cell_type":"code","source":"##joins\n\nx=['a','b','c','d']\n' '.join(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f80b33a144ab40c470707a1a60646005180b4678"},"cell_type":"code","source":"input_string=' '.join(data['question_text'])\nwc=WordCloud().generate(input_string)\nplt.imshow(wc)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"908e83e2c9c94604cc83b0a63f87809c4cdb294b"},"cell_type":"markdown","source":"1. text cleaning --only done on supervised sentiment analysis\n1. 1.  convert every character to lower case\n1. 2. remove junk characters\n1. 3.  remove commonlu used words using nltk lib\n1. 4. identify root words using stemmer\n"},{"metadata":{"trusted":true,"_uuid":"6258bb0df2123daa99c2fcde250b2ac861f1d62d"},"cell_type":"code","source":"docs= data['question_text'].str.lower().str.replace('[^a-z ]','')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"10db65a0d18589e0a1b6daed22f3546cf82e4e5a"},"cell_type":"code","source":"docs.head(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d3be533d49702783eb8edcf5023e6428f784d26"},"cell_type":"code","source":"stopwords=nltk.corpus.stopwords.words('english')\nstopwords[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0785672a452823aadb900e3989abd6758de6e976"},"cell_type":"code","source":"stemmer= nltk.stem.PorterStemmer()\nstemmer.stem('playing')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59e8ff6b57c05560d9658ba38ea5d91049eb5321"},"cell_type":"code","source":"docs_clean=[]\n\nfor doc in docs:\n    words=doc.split(' ')\n    words_clean=[]\n    \n    for word in words:\n        if word not in stopwords:\n            words_clean.append(stemmer.stem(word))\n    doc_clean=(' '.join(words_clean))\n    docs_clean.append(doc_clean)\n\nprint(docs_clean)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"211131c4cefa47aaf6deb3779f05a418e38ce1ee"},"cell_type":"code","source":"### using list comprehension\n\ndocs = imdb['review'].str.lower().str.replace('[^a-z ]','')\ndef clean_sentence(text):\n    words = text.split(' ')\n    words_clean = [stemmer.stem(word) for word in words if word not in stopwords]\n    return ' '.join(words_clean)\ndocs_clean = docs.apply(clean_sentence)\ndocs_clean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8ebbb78c40be35d8bbee5a2f76dd0f4e5b81e53f"},"cell_type":"code","source":"######Properties of document term matrix\n\n### every row is a document and each document should be represented as vectors\n### size of vector can be identified by the no. of unique terms in the corpus(i.e. all the reviews together) \n#--it is the no. of columns in the document term matrix\n## column sum will give frequency of a word across all reviews/documents\n## row sum will give no. of unique words in a review/document(document length)\n## sparse matrix ==> most of the values are 0 ==> sparcity = (no. of zero's)/(no. of rows*no.of columns)\n## high dimension data\n## every column is a vector representation of a term","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"603e1f02e8732ef88c6895e1a4b36250e4dfb904"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer\n\nvectorizer = CountVectorizer()\nvectorizer.fit(docs_clean)\ndtm = vectorizer.transform(docs_clean)\ndtm\n## in o/p 748 - no. of rows, 2475 - no. of columns(no. of unique words), 6797 are the non zero values \n## here the output is stored as a compressed matrix, as we might get memory errors\n## (748*2475) - 6797 is the no. of zero's","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dac3cf37f55689b8d39a68056feacb6031a21bb3"},"cell_type":"code","source":"no_of_zeroes = (748*2475) - 6797\nsparcity = no_of_zeroes / (748*2475) * 100\nsparcity","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07bb8a805d04d7eb0b2fef031aae145779865a0e"},"cell_type":"code","source":"### to display the document term matrix\ndf_dtm = pd.DataFrame(dtm.toarray(), columns=vectorizer.get_feature_names())\n(df_dtm == 0).sum() ## column wise no. of zeros\n(df_dtm == 0).sum().sum() ## total zeros in the dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d6f9c7d5a720ec9d6ac1d707a4eab64628ab770"},"cell_type":"code","source":"df_dtm.sum().sort_values(ascending=False).head(2)## frequency of each word","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a5ee95405562828affad764c15d44a3b23e7e146"},"cell_type":"code","source":"df_dtm.sum(axis = 1).sort_values(ascending = False).head(1) ##uniques words per row/document","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f90e8a31c9da0abca711bc2d2f7c99e4fdf39e5d"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ntrain_x, validate_x = train_test_split(df_dtm, test_size = 0.2, random_state = 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63c0c6db723fec7a7acc8dde5a91acb0c8258afc"},"cell_type":"code","source":"train_y = data.iloc[train_x.index]['target']\nvalidate_y = dat.iloc[validate_x.index]['target']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"311ad4fb130eff2d1fd632268f052abb95fd5668"},"cell_type":"code","source":"from sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import accuracy_score\nrf_model = RandomForestClassifier(random_state=100,n_estimators=300)\nrf_model.fit(train_x,train_y)\nrf_predict = rf_model.predict(validate_x)\naccuracy_score(validate_y, rf_predict)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b31423641003675d075fee94e9ff3daeb8915d6"},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\nfrom sklearn.metrics import accuracy_score\nNB_model = MultinomialNB(alpha=1)\nNB_model.fit(train_x,train_y)\nNB_predict = NB_model.predict(validate_x)\naccuracy_score(validate_y, NB_predict)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}