{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\ndata=pd.read_csv('../input/train.csv')\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"data.shape\nfrom matplotlib import *\nimport matplotlib.pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a2dc21ee5370a9369500821e8530ab24cad6749"},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"345b23c0085e895274ffc4551653749d2aef9967"},"cell_type":"code","source":"import nltk\nimport wordcloud","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"94cb90c51d71d7cd8e300fb5d486b2493f57da6a"},"cell_type":"code","source":"class_0=data[data['target']==0]\nclass_1=data[data['target']==1]\nwc=wordcloud.WordCloud().generate(' '.join(class_0['question_text']))\nplt.imshow(wc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a18d983c3a41155cb54f4a5f34b689c17ded97ee"},"cell_type":"code","source":"wc1=wordcloud.WordCloud().generate(' '.join(class_1['question_text']))\nplt.imshow(wc1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4408a5dbcf4d8b1ff0423077dd40006846d09675"},"cell_type":"code","source":"nltk.corpus.stopwords.words('english')\nstop_words=nltk.corpus.stopwords.words('english')\njunk_words=[\"amp\",'rt','https','will']\nlen(stop_words)\nstop_words.extend(junk_words)\nlen(stop_words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cec7fe4bb5ed08dad212bd80ee54f4c229d1bb1e"},"cell_type":"code","source":"## cleaning the data\ndocs=data['question_text'].str.lower()\ndocs.head()\ndocs=docs.str.replace('[^a-z #@]','') # retain all alphabets with #@\ndocs.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e35edc2c0ca5473e55f5890f3946c8df22db99bc"},"cell_type":"code","source":"stemmer=nltk.PorterStemmer()\ndef clean_text(row_text):\n    #print(type(row_text))\n    row_words=row_text.split(' ')\n    #print(row_words)\n    row_words= [stemmer.stem(word) for word in row_words if word not in stop_words]\n    #print(row_words)\n    #print('----')\n    return ' '.join(row_words)\n\ndocs_clean=docs.apply(lambda v: clean_text(v))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8e1225035abbb6ba269c88f43fdc8d8d7cebcd48"},"cell_type":"code","source":"data[\"clean_one\"]=docs_clean","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1fa625a9d0b81a7bc5f3d7616eb80529aea2a0dc"},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4afd51a7c8972550ccc00d7acd50724ab6c8202"},"cell_type":"code","source":"df=data[[\"clean_one\",\"target\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"024435f4ef53f7abf8aee314e2b1e585c430eccd"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95c9672bbde992de4df4e46ade50923b55bbe8c7"},"cell_type":"code","source":"\nfrom sklearn.model_selection import train_test_split\n\ntrain,validate= train_test_split(docs_clean,test_size=0.3, random_state=100)\ntrain_y=data.loc[train.index][\"target\"]\nvalidate_y=data.loc[validate.index][\"target\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0f900f28e8ea0a8210a9d697e62292a68f3285e3"},"cell_type":"code","source":"train.shape, validate.shape,train_y.shape,validate_y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"473380e3729d504e627c9a9ca6f4310efe929fbf"},"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ccba32acb400296f0c65da727580328e88cfd7d9"},"cell_type":"code","source":"cv=CountVectorizer()\ncv.fit(train)\ntrain_x_sparse=cv.transform(train)\nvalidate_x_sparse=cv.transform(validate)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"603fc90bc73a4e414a4dc385b6c00ba575a094d5"},"cell_type":"code","source":"from sklearn.tree import DecisionTreeClassifier\nfrom sklearn.ensemble import RandomForestClassifier\ndt_model=DecisionTreeClassifier(max_depth=20,random_state=100)\ndt_model.fit(train_x_sparse,train_y)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4ac96f8c3c2c7a1cb1e2cdb83dd423c4f695ff91"},"cell_type":"code","source":"pred_class=dt_model.predict(validate_x_sparse)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"37051c81e90a13ddb6f07362aaad47fd163081f5"},"cell_type":"code","source":"from sklearn.metrics import accuracy_score,f1_score,roc_curve,auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22dcd48e3b22e4e0ed51850b2caf245c65dc5e48"},"cell_type":"code","source":"print(accuracy_score(validate_y,pred_class))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfba3e2aebea4c6d335ad055a2c06b65babbdcd8"},"cell_type":"code","source":"print(f1_score(validate_y,pred_class))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f85ef02aaab75d0aaf6b824435af915437b9352e"},"cell_type":"code","source":"pred_probs=pd.DataFrame(dt_model.predict_proba(validate_x_sparse),columns=['Sincere','Insincere'])\npred_probs.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"251b32e5e0898eeae51c307b66a5d22848a69528"},"cell_type":"code","source":"fpr,tpr,thresholds=roc_curve(validate_y,pred_probs[\"Insincere\"])\nauc_dt=auc(fpr,tpr)\nplt.plot(fpr,tpr)\nplt.legend([\"Decision Tree -AUC: %2f\" % auc_dt])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8e6592834ec683090e7fb66232a043bf6d558b6"},"cell_type":"code","source":"test=pd.read_csv('../input/test.csv')\ntest_docs=test['question_text'].fillna('').str.lower()\ntest_docs=test_docs.str.replace('[^a-z #@]','')\ntest_docs_clean=test_docs.apply(lambda x: ' '.join([stemmer.stem(word) for word in x.split(' ') if word not in stop_words]))\ntest_docs_clean.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93b70cc239653441624634b25dd4ab8cc9a2df5d"},"cell_type":"code","source":"test_x=cv.transform(test_docs_clean)\ntest_pred_class=dt_model.predict(test_x)\ntest_pred_class.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7113fc779f1d78d50f6dc0dfb83ecbda948f565"},"cell_type":"code","source":"submission=pd.DataFrame({'qid':test['qid'],'prediction':test_pred_class})\nsubmission.to_csv(\"submission.csv\",index=False)\nsubmission.head() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1839c8510c9486711a7486446cb26ec48128ee77"},"cell_type":"code","source":"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f4a2458507d27a9a4af8dfe5dd4e8f054eae477e"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7be22b2fff81e3ff75df4b682663ac0f1cfeb5f3"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f4e56e42add36b26c23137045eb78c6a35ccba1"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9ddfd0eed46dd072322b07d2193c97fbe65689b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}