{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"**Notebook Objective:**\nThe objective of this notebook is to explore how the data looks without any processing.\n\n**Competition Objective :**\nThe main objective of this competition is to predict whether the asked question on Quora is sincere or not. "},{"metadata":{"_uuid":"a92a6b59d4c1480a577247d7af5bc5a01df66e60"},"cell_type":"markdown","source":"**Import all required libraries**"},{"metadata":{"trusted":true,"_uuid":"a20ab29c84033e2e9aa8bb44591cd13a5002b86a"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom plotly import tools\nimport plotly.offline as py\nimport plotly.graph_objs as go\nfrom nltk.corpus import stopwords \nfrom nltk import word_tokenize, sent_tokenize, pos_tag, ne_chunk, FreqDist\nfrom textblob import TextBlob\nimport collections\nfrom sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\nfrom sklearn.preprocessing import normalize\nfrom wordcloud import WordCloud\n%matplotlib inline\npy.init_notebook_mode(connected=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"429c311b55b662b40abdfb95600ad79ca764fa87"},"cell_type":"markdown","source":"**Load data**"},{"metadata":{"trusted":true,"_uuid":"dc47f49bb98d90baaa49cf490268686ab7df16df"},"cell_type":"code","source":"train_data=pd.read_csv(\"../input/train.csv\")\ntest_data=pd.read_csv(\"../input/test.csv\")\nsubmission_data=pd.read_csv(\"../input/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"785a3a10e026530c759298773f6cb458554ec2ea"},"cell_type":"markdown","source":"**Inspect the data**\n\nThe training data set has a total of 1306122 records with three columns. testing data has a total of 56370 records with 2 columns. Submission data has a total of 56370 with 2 columns"},{"metadata":{"trusted":true,"_uuid":"131b8dcd3d3b342c3ef4cfbbd530667f10819593"},"cell_type":"code","source":"train_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ca57df7d3ce9d91833dd48baa31f30368c5b2ae4"},"cell_type":"code","source":"test_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"57e4d1a2971db2a21f88738a56ea4cdedd95dab5"},"cell_type":"code","source":"submission_data.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4d01c9d087b74ac34cfcb0bb16df498fe8b11dfc"},"cell_type":"markdown","source":"**Focus on Training data**"},{"metadata":{"trusted":true,"_uuid":"681d5a7fe99f01389b4c59bd616d70d582926e4c"},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff1a6b3e9838b847b704c1079b5836a74798cf6b"},"cell_type":"code","source":"train_data.columns","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"714c41f9dd09eaf42c7dbfd0917c754046f69f28"},"cell_type":"markdown","source":"**Check for null values**"},{"metadata":{"trusted":true,"_uuid":"2569dc650f622c92bf9dac3c3ddcf853156bd3b2"},"cell_type":"code","source":"train_data['question_text'].isnull().value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42c0c9aabd6c7e4b446b0452f41e9745ea2a6aca"},"cell_type":"code","source":"train_data['target'].isnull().value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b6df08c497c320c3b395b1eeee3861499ea77229"},"cell_type":"code","source":"train_data['question_text'].drop_duplicates().count()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f61f964918e8f78f6d670cb994a7c4dc2a259904"},"cell_type":"code","source":"# Unique target\ntrain_data['target'].unique()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93207c31cf4407e9e705ebdd317258621246c740"},"cell_type":"code","source":"insicere_quiz=train_data[train_data['target']==1]\nsincere_quiz=train_data[train_data['target']==0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e1c55239fc0020221191ee72d4f464fa5f89291b"},"cell_type":"markdown","source":"**Percentage of insincere questions  in the dataset**"},{"metadata":{"trusted":true,"_uuid":"fb2959fa3f35519aec1c5739d4e4cbb5746189c1"},"cell_type":"code","source":"insicere_quiz['target'].count()/train_data['target'].count() * 100","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"051a3a438bd634bacef143b53d1cc210f80acdb9"},"cell_type":"markdown","source":"**Percentage of sincere questions in the dataset**"},{"metadata":{"trusted":true,"_uuid":"ee616ac1f18ef48ac2dc6d1aad76ce3d8570983f"},"cell_type":"code","source":"sincere_quiz['target'].count()/train_data['target'].count() * 100","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e6ac1dd3a0cc40eb1b2bae9312eae4b558e1242"},"cell_type":"code","source":"target_counts = train_data['target'].value_counts()\ntarget_counts","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","collapsed":true,"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"markdown","source":"**Simple visualization for target distribution**"},{"metadata":{"trusted":true,"_uuid":"cf5c500d5ba2c5f6b62066b24e638a352dc64754"},"cell_type":"code","source":"pie_labels = (np.array(target_counts.index))\npie_sizes = (np.array((target_counts / target_counts.sum())*100))\n\ntrace = go.Pie(labels=pie_labels, values=pie_sizes)\npie_layout = go.Layout(title='Target distribution',font=dict(size=16),width=500,height=500)\nfig = go.Figure(data=[trace], layout=pie_layout)\npy.iplot(fig, filename=\"file_name\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b339f22af8c862e0c97cf8b1312f910a63674ae"},"cell_type":"code","source":"bar_graph = go.Bar(\n        x=target_counts.index,\n        y=target_counts.values,\n        marker=dict(\n        color=target_counts.values,\n        colorscale = 'Picnic',\n        reversescale = True\n    ),\n)\n\nbar_layout = go.Layout(title='Target Distrinution',font=dict(size=20))\nfig = go.Figure(data=[bar_graph], layout=bar_layout)\npy.iplot(fig, filename=\"file_name\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ec503c9ef2ca07cbfca36af739e163f60a1af78a"},"cell_type":"markdown","source":"**Simple Natural Language Processing (NLP) Tasks**"},{"metadata":{"_uuid":"52922700c7d9132b9f194534ff285f81b6974530"},"cell_type":"markdown","source":"**1. Checking for stop words**"},{"metadata":{"trusted":true,"_uuid":"d7b0b6edc251345ca6e7afd7584dc616975a0265"},"cell_type":"code","source":"stop_words = stopwords.words('english')\n\ntokens=[]\nfor i in train_data[0:10]['question_text']:\n    for j in word_tokenize(i):\n        tokens.append(j)\n\nfiltered_text = [token for token in tokens if not token in stop_words]  \n\nfiltered_text = [] \n  \nfor i in tokens: \n    if i in stop_words: \n        filtered_text.append(i) \n        \nprint(np.array(filtered_text))\nlen(np.array(filtered_text))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5122b27f3cd2e7223a70ddc0254f7c96c44f5cf5"},"cell_type":"markdown","source":"**2. Part Of Speech Tagging (POS)**"},{"metadata":{"trusted":true,"_uuid":"2c3215ab1140b26f351ee93933cba047e4e68648"},"cell_type":"code","source":"token=[]\nfor i in train_data[0:2]['question_text']:\n    token.append(pos_tag(word_tokenize(i)))\n\nprint (token)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"926550f69a573918d5eec9f5d3f62c48e468c4f0"},"cell_type":"markdown","source":"**3. Extracting Noun Phrases**"},{"metadata":{"trusted":true,"_uuid":"d7301803da1524cd36defc52ac66fb0cb3d7c9e4"},"cell_type":"code","source":"for i in train_data[0:10]['question_text']:\n    print(TextBlob(i).noun_phrases)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4ab6eb4e76e138a0d3d85689345b6de170e290c3"},"cell_type":"markdown","source":"**4. Detecting language**"},{"metadata":{"trusted":true,"_uuid":"93d46c7087ae376fcc180fd7091b7dee9981c421"},"cell_type":"code","source":"# lng=[]\n# for i in train_data[0:5]['question_text']:\n#     lng.append(TextBlob(i).detect_language())\n    \n# set(lng)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e0aecf3c998cdcb3761e04a26ec2c318be6730df"},"cell_type":"markdown","source":"**5. Word frequency**"},{"metadata":{"trusted":true,"_uuid":"66dbaffc7254b7e3719c3d5e2333ff714797c6a0"},"cell_type":"code","source":"for i in train_data[0:5]['question_text']:\n    print(TextBlob(i).word_counts[\"quebec\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"255c53c3acc1a59bd82173d3129602f52e271803"},"cell_type":"markdown","source":"**6. Words count**"},{"metadata":{"trusted":true,"_uuid":"f5900ff80f7cf13393429d11829b4a6f9aa225a0"},"cell_type":"code","source":"for i in train_data[0:10]['question_text']:\n    print(i,\" => \",len(word_tokenize(i)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73241e5801c57e51c455bb1015c14c95925528a3"},"cell_type":"markdown","source":"7. Sentence count"},{"metadata":{"trusted":true,"_uuid":"8c74d509c728e86a4217ec7503e2828fccc42590"},"cell_type":"code","source":"for i in train_data[0:10]['question_text']:\n    print(i,\" => \",len(sent_tokenize(i)))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3ba9a9f1f1b33f8b548fc59192e8f9b9a1c1d3dd"},"cell_type":"markdown","source":"8. Most Common Words"},{"metadata":{"trusted":true,"_uuid":"b8601e7ee548a5fbdb01432781088a89b9b17191"},"cell_type":"code","source":"tokens=[]\nfor i in train_data[0:10]['question_text']:\n    for j in word_tokenize(i):\n        tokens.append(j)\n\n\nfrequency_distribution=FreqDist(tokens).most_common()\nprint(frequency_distribution)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40855e6fc4f9cbec4ff2c45d86d44cf7961298c0"},"cell_type":"code","source":"tokens=[]\nfor i in train_data[0:10]['question_text']:\n    for j in word_tokenize(i):\n        tokens.append(j)\n\n\nfrequency_distribution=FreqDist(tokens)\nword_c={}\nfor i in frequency_distribution:\n    word_c[i]=token\n    word_c[i]=frequency_distribution[i]\n\nsorted(word_c.items(), key=lambda x: x[1], reverse=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5068a2b6e233dbde604be2b82a775082bf3e06e"},"cell_type":"markdown","source":"**9. WordCloud**"},{"metadata":{"trusted":true,"_uuid":"299b2b988c439aeebf9b85f08724ab54c0938bba"},"cell_type":"code","source":"text=str(train_data[0:1000]['question_text'])\nwordcloud = WordCloud(width=1600, height=800).generate(text)\nplt.figure( figsize=(20,10), facecolor='k')\nplt.imshow(wordcloud)\nplt.axis(\"off\")\nplt.figure( figsize=(40,30) )\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"036b876d35c41e4761d48bb7b06c88a5ecb1a14d"},"cell_type":"markdown","source":"**9. Sentiment Analysis**"},{"metadata":{"trusted":true,"_uuid":"bb4ec37addad87a0fffabfe4745abc542d7e4c0b"},"cell_type":"code","source":"# Calculating Sentment Analysis with TextBlob\nfor i in train_data[0:5]['question_text']:\n    print(i,\" => \",TextBlob(i).sentiment)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c954ed55ae57fdd014cb3e433bd99f1afbc1dd39"},"cell_type":"code","source":"# Extracting the sentiment polarity of a text\nfor i in train_data[0:5]['question_text']:\n    print(TextBlob(i).sentiment.polarity)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b55e1e45ea6bfdd457b68649ea70185076c1a95c"},"cell_type":"code","source":"# Extracting the sentiment subjectivity of a text\nfor i in train_data[0:5]['question_text']:\n    print(TextBlob(i).sentiment.subjectivity)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"917e9a19ad1b437b964d13090b7804270aee34dd"},"cell_type":"markdown","source":"**10. N-grams (tri-gram)**"},{"metadata":{"trusted":true,"_uuid":"865201bb1b2c8ca4c84150ce5dddc1d6e9854706"},"cell_type":"code","source":"for i in train_data[0:2]['question_text']:\n    print(TextBlob(i).ngrams(n=3))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ba0e01e7e7b781bab6ee6e80d9958f78a65396e"},"cell_type":"markdown","source":"**11. Term Frequency Inverse Document Frequency (tf-idf)**"},{"metadata":{"trusted":true,"_uuid":"8394f5413667cd2bae0a8c1ccdc2c7dc6aa3ea89"},"cell_type":"code","source":"corpus=[]\nfor i in train_data[0:5]['question_text']:\n    corpus.append(i)\n\ncvect = CountVectorizer(ngram_range=(1,1))\ncounts = cvect.fit_transform(corpus)\nnormalized_counts = normalize(counts, norm='l1', axis=1)\n\ntfidf = TfidfVectorizer(ngram_range=(1,1), smooth_idf=False)\ntfs = tfidf.fit_transform(corpus)\nnew_tfs = normalized_counts.multiply(tfidf.idf_)\n\nfeature_names = tfidf.get_feature_names()\ncorpus_index = [n for n in corpus]\ndf = pd.DataFrame(new_tfs.T.todense(), index=feature_names, columns=corpus_index)\n\nprint(df)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ce6849618c57147f3eb90d2d6e331e908f4420fb"},"cell_type":"markdown","source":"**12. Bag of Words (BoW)**"},{"metadata":{"trusted":true,"_uuid":"1405034eb5e7ec446d8d0e7592ce35e6411557eb"},"cell_type":"code","source":"#Bow with collection\ntoken=[]\nfor i in train_data[0:5]['question_text']:\n    token.append(i)\n\nbow = [collections.Counter(words.split(\" \")) for words in token]\ntotal_bow=sum(bow,collections.Counter())\nprint(total_bow)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5703fde49b27373d2c87ae1a6eec73a5340073d4"},"cell_type":"markdown","source":"**To Continue .....**"}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}