{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\ndata=pd.read_csv('../input/train.csv')\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"docs=data['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2d517c63d3f4f803fd7619e202cd2c2dab0bda2e"},"cell_type":"code","source":"from wordcloud import WordCloud\nimport matplotlib.pyplot as plt\nwc=WordCloud(background_color='white').generate(''.join(docs))\nplt.imshow(wc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6caf7812aac249a038d78c76bcf4a9d6bd7f374b"},"cell_type":"code","source":"is_sincere = data['target']==0\nnot_sincere = data['target']==1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d9686582c68ae78f0935365d63818133999bf62"},"cell_type":"code","source":"docs0 = data[is_sincere]\ndocs1 = data[not_sincere]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63bb77027c1ef1c4775c3ad21603d9c24906cb8a"},"cell_type":"code","source":"print(docs0.head())\ndocs1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"862d496ff71a049b8f05fd83bdb6f6eef8493513"},"cell_type":"code","source":"dc0 = docs0['question_text']\nwc = WordCloud(background_color='white').generate(' '.join(dc0))\nplt.imshow(wc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd43e863e09cd619b329edd6f19121950a158423"},"cell_type":"code","source":"dc1 = docs1['question_text']\nwc1 = WordCloud(background_color='white').generate(' '.join(dc1))\nplt.imshow(wc1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8077e80fad518ddabfc46c7b0e0dd8a4a4f65a79"},"cell_type":"code","source":"import nltk\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db2f3275c5549b7f5ddb796ec003dc665c6996e8"},"cell_type":"code","source":"stopwords = nltk.corpus.stopwords.words('english')\nlen(stopwords)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d59cff8ab50feda2ab79a6e61eeab3dd447237a"},"cell_type":"code","source":"docs = docs.str.lower()\ndocs.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1d27d1b9c64f6b5b968c1c347d28323bb38d9152"},"cell_type":"code","source":"docs.str.replace('[^a-z ]','')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ad17af2244134d26252343fd353716053c5cdd90"},"cell_type":"code","source":"stemmer = nltk.stem.PorterStemmer()\n\ndef clean_sentence(doc):\n    words = doc.split(' ')\n    words_clean = [stemmer.stem(word) for word in words if word not in stopwords]\n    doc_clean = ' '.join(words_clean)\n    return doc_clean\n\n\ndocs_clean = docs.apply(clean_sentence)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f68d9b1f4338f36139d7a153ef385c594e5754de"},"cell_type":"code","source":"train_x, test_x, train_y, test_y = train_test_split(docs_clean, \n                                                    data['target'],\n                                                    test_size = 0.2,\n                                                    random_state=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f70031399ba0702d9a982d49d219f36806e69c40"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\ntfidf = TfidfVectorizer(min_df=50).fit(train_x)\ntrain_x = tfidf.transform(train_x)\ntest_x = tfidf.transform(test_x)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7aef5731ab7a07195a2928c460c026f2ad3601fc"},"cell_type":"code","source":"from sklearn.naive_bayes import GaussianNB, MultinomialNB, BernoulliNB\nfrom sklearn.metrics import accuracy_score, f1_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b7911b9e4e3876122ffbd37a773ac5719ee1c603"},"cell_type":"code","source":"model_mnb = MultinomialNB().fit(train_x,train_y)\ntest_pred = model_mnb.predict(test_x)\nprint(accuracy_score(test_y,test_pred))\nprint ('F1 score:', f1_score(test_y, test_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a609527dee81ee3f5d2b9dde34c757ad095f7d33"},"cell_type":"code","source":"model_bnb = BernoulliNB().fit(train_x,train_y)\ntest_pred = model_bnb.predict(test_x)\nprint(accuracy_score(test_y,test_pred))\nprint ('F1 score:', f1_score(test_y, test_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"78e2dd39d65c12fb905581cb7d33a863e5681d3e"},"cell_type":"code","source":"train_x, test_x, train_y, test_y = train_test_split(docs_clean, \n                                                    data['target'],\n                                                    test_size = 0.2,\n                                                    random_state=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e07f385ff55364419d37be8c225958da9f939eb"},"cell_type":"code","source":"vectorizer = CountVectorizer(min_df=50).fit(train_x)\ntrain_x = vectorizer.transform(train_x)\ntest_x = vectorizer.transform(test_x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a4010841341f0e33ea5cdfd200ec576895b0c56"},"cell_type":"code","source":"model_bnb = BernoulliNB().fit(train_x,train_y)\ntest_pred = model_bnb.predict(test_x)\nprint(accuracy_score(test_y,test_pred))\nprint ('F1 score:', f1_score(test_y, test_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36a7258b4e30691db351348a71659da99e1548db"},"cell_type":"code","source":"model_mnb = MultinomialNB().fit(train_x,train_y)\ntest_pred = model_mnb.predict(test_x)\nprint(accuracy_score(test_y,test_pred))\nprint ('F1 score:', f1_score(test_y, test_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3820dec2c218ed99145a2c3389a4c58004fc9c22"},"cell_type":"code","source":"test_df = pd.read_csv('../input/test.csv')\ntest_df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8b9e24f11de24ff34096e6c61227224c71946224"},"cell_type":"code","source":"test_docs = test_df['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"174bd1a696d85d6013df370c9f4015a00f07fe85"},"cell_type":"code","source":"test_docs = test_docs.str.lower()\ntest_docs.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"14765ae8ae821467011aa1d869b3f0372c438f64"},"cell_type":"code","source":"test_docs_clean = test_docs.str.replace('[^a-z ]','')\ntest_docs_clean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d0e158468378e7f8da49c80318d469838ce1bc0"},"cell_type":"code","source":"test_docs_clean.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9487e1616cdb958c6ec2058aeec87cc20bc8cc74"},"cell_type":"code","source":"test_df_clean = vectorizer.transform(test_docs_clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e409ea3a5c0a05bafc2fc07ca46862464ea9067"},"cell_type":"code","source":"test_df_pred = model_mnb.predict(test_df_clean)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f3acc4e439f2c684509dd10fb4b7e95100313ea8"},"cell_type":"code","source":"test_df_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73409a92eaddeb16438cf65e00265893b401c87b"},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c1696112c095117ad525f4b4957494f15d7f39bd"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2105294d7a566364d305b6be063816b37244cfa2"},"cell_type":"code","source":"mysub = pd.DataFrame(test_df['qid'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3603f0c8c97452f6b19258dcd27597c213b978f6"},"cell_type":"code","source":"test_df_pred = pd.DataFrame(test_df_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"442f43bdbbea734c0673b38fad070e779849ef21"},"cell_type":"code","source":"final_sub = pd.concat([mysub,test_df_pred],axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e746ecc73f10f571b673b13499e470b8412ea95c"},"cell_type":"code","source":"final_sub.columns=['qid','prediction']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d22f5d1541712b83eed823ab74c8e9d12f08255d"},"cell_type":"code","source":"final_sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e4f68a1718f86e1cc7710def530bc0ebc484872"},"cell_type":"code","source":"from pandas import DataFrame","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"002a13d50bcc98fb50d948fd088a15f8b1971640"},"cell_type":"code","source":"final_sub.to_csv(\"submission.csv\",index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"414a93e0ab3a2d4fbe2d9dca07eb71a6f060e86e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}