{"cells":[{"metadata":{"trusted":false,"_uuid":"ac5fe1cf89a4b0f3c7eee803be36dc3acca64a6b"},"cell_type":"code","source":"import pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn.metrics import classification_report\n\n%matplotlib inline\n\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5d2f1fffa15840ecfdc587d0a9ea0076e2b90ab9"},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a1ccbcbd5cbfd59e40eb50a59869e15fc53fb21d"},"cell_type":"code","source":"df.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"be1ed9cd7a74ca28b3c1bdf99e00c1edd520b7d9"},"cell_type":"code","source":"df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"22ae790e64c1fc132043ce2149117c1103859a5f"},"cell_type":"code","source":"df['target'].value_counts() #normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"6c2a2a3f42deee00c2a043b179ff62c98d1bb745"},"cell_type":"code","source":"toxic = df.loc[df['target'] == 1, 'question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"85d81b41ae6ce310fee3222c010e16ed74e1da33"},"cell_type":"code","source":"toxic.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"df0b61f6d3817a3594f4459e870c5e4113443c9f"},"cell_type":"code","source":"words = df['question_text'].str.split()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"34ee283f932c2f360b4279465c959b56e46dd261"},"cell_type":"code","source":"words.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"cc6a4a205390b1abf3f2678482d49d93cf7d3c14"},"cell_type":"code","source":"num_words = words.apply(len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"5c57aa53d750d483b6492a4621afae2e699a0af7"},"cell_type":"code","source":"num_words.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8c6f2c344183bc27c22e4013ef4d1529cf889f8b"},"cell_type":"code","source":"sns.distplot(num_words[df['target'] == 0], label='normal')\nsns.distplot(num_words[df['target'] == 1], label='toxic')\nplt.xlabel('num. words')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a71410499aec7e53f8bbc2f032ad1c44ed94fa17"},"cell_type":"code","source":"pd.crosstab(num_words, df['target'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"a41697cda2ce1ea0472a6bd604afe6112ee8b051"},"cell_type":"code","source":"first_word = list(map(lambda l: l[0], words))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"7aeeb3561b31643bde203c5374f1ce3471ab5d3e"},"cell_type":"code","source":"first_word[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"189ee27b50a7310fdf823a24cca152902a6077bc"},"cell_type":"code","source":"pd.Series(first_word).value_counts().head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"bd42428cbbf107f43d5d84431eab74aa42e5219e"},"cell_type":"code","source":"pd.Series(first_word).groupby(df['target']).value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8b88fd74fe7b3f9a3a8ea1d4dda1c8b242fd0340"},"cell_type":"code","source":"first_character = list(map(lambda w: w[0], first_word))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"290f40f38e8d9b9a14ad282bbf4522b94208058c"},"cell_type":"code","source":"pd.Series(first_character).groupby(df['target']).value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"c0caa4c10dc9da1b155f81de7fd095f11ed7d599"},"cell_type":"code","source":"print(classification_report(df['target'], num_words == 1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"8e2545cdb53637dba785e3571d5f2ace4a7fdd2b"},"cell_type":"code","source":"print(classification_report(df['target'], num_words >= 30))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"84107f51282bb896307155c20aa1f26710d051bf"},"cell_type":"code","source":"test_df = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"65099addb1c7ed95fa918b8a8b160bc8aaa3c51f"},"cell_type":"code","source":"test_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"b42bf5a5e951ef81899a8c2c048bbbe4e934fa62"},"cell_type":"code","source":"words_test = test_df['question_text'].str.split()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"2ed669305bab65478cae79d84baa1c54fe71e801"},"cell_type":"code","source":"num_words_test = words_test.apply(len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"15e88219b19aecb9be23129ccab0e5ba5343bbad"},"cell_type":"code","source":"out = pd.DataFrame({'qid': test_df['qid'], 'prediction': (num_words_test >= 30).apply(int)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"00f7b53592631a78e6c1715c8c761912bd7d2df1"},"cell_type":"code","source":"predicting_toxic = test_df.loc[out['prediction'] == 1, 'question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"133742d1ffd469145179ce800b60cebb91c8681f"},"cell_type":"code","source":"predicting_toxic.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"463404aa82bc7838d3ccf325fdd3f869d594af23"},"cell_type":"code","source":"out.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false,"_uuid":"4895c234a8ec41a3c1316146e53373c6e7df2032"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":1}