{"cells":[{"metadata":{"_uuid":"3eea589a3416ae470b147b542aec5653d6c49c65"},"cell_type":"markdown","source":"I'm beginner at NLP, so I learn a lot from kernel. Thank you for post great kernel!<br>\nThis is my cheap approach, \"Which word make possibility of insincere sentence high?\""},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('../input/train.csv')\ntest_data = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"54b116cfa7f6eb6c0e6644f453063a5898ff71c4"},"cell_type":"code","source":"train_text = train_data['question_text']\ntest_text = test_data['question_text']\ntrain_target = train_data['target']\nall_text = train_text.append(test_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a4622583429c4c340497c516705820c6ced11f3"},"cell_type":"code","source":"tfidf_vectorizer = TfidfVectorizer()\ntfidf_vectorizer.fit(all_text)\n\ntrain_text_features_tf = tfidf_vectorizer.transform(train_text)\ntest_text_features_tf = tfidf_vectorizer.transform(test_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"963cb73b931de1aa2ce416d005a9369d8006c199"},"cell_type":"code","source":"target0_index = list(train_data.query(\"target == 0\").index)\ntarget1_index = list(train_data.query(\"target == 1\").index)\nprint(len(target0_index))\nprint(len(target1_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ed3f65300f61b82897df7dbe6fd40999e9524a9f"},"cell_type":"code","source":"# check the difference between target1 and target0\ntarget0_score = train_text_features_tf[target0_index]\ntarget1_score = train_text_features_tf[target1_index]\n\ndiff_score = np.mean(target1_score, axis=0) - np.mean(target0_score, axis=0)\ndiff_ary = np.argsort(-diff_score).tolist()[0]\nprint(diff_score)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0c2dbf482ccfc4746b56585561a7a32b390ab1f"},"cell_type":"code","source":"# insincere words\nfor i in range(30):\n    insincere_word = tfidf_vectorizer.get_feature_names()[diff_ary[i]]\n    insincere_data = train_data[train_data['question_text'].str.contains(insincere_word)]\n    print(\"======================\")\n    print(insincere_word)\n    print(\"len: {}\".format(len(insincere_data)))\n    print(\"mean: {}\".format(np.mean(insincere_data[\"target\"])))\n    print(\"======================\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cc75512b53385ea53afb8cd1bb292382a1f0c9e8"},"cell_type":"code","source":"# NOT insincere words\nfor i in range(30):\n    insincere_word = tfidf_vectorizer.get_feature_names()[diff_ary[-(i+1)]]\n    insincere_data = train_data[train_data['question_text'].str.contains(insincere_word)]\n    print(\"======================\")\n    print(insincere_word)\n    print(\"len: {}\".format(len(insincere_data)))\n    print(\"mean: {}\".format(np.mean(insincere_data[\"target\"])))\n    print(\"======================\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"770876a6444df6beb493422928dfd1d14f205252"},"cell_type":"markdown","source":"Below words may be insincere:\n\n* Race\n* Sex\n* skin color\n\nBelow words might not be insincere:\n* work (job, work, learn, business, ... engineering is very low! :D )\n* positive word(good, best)\n\n"},{"metadata":{"trusted":true,"_uuid":"5e8b94b149f38afbef338a5ccea6373f4b65a525"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}