{"cells":[{"metadata":{"_uuid":"d2130cb1d1f7d31595f548a443f9101cfc92ae2f"},"cell_type":"markdown","source":"## Analyse chinese text in the data\n"},{"metadata":{"trusted":true,"_uuid":"6ad40f5fbbf54c91d9dbf312143922928c7738a1"},"cell_type":"code","source":"import pandas as pd\nimport re\n\n# Display whole text of dataframe field and don't cut it\npd.set_option('display.max_colwidth', -1)    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"017163e18a123059a294c4fca0e63f23db81bc41"},"cell_type":"markdown","source":"#### Load data"},{"metadata":{"trusted":true,"_uuid":"794db32b1a665b842dd9ed8ff59ddc9ffb38032e"},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")\ndf = pd.concat([df_train ,df_test],sort=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"03a97dab59cbb53019f1693e1e35d4b948d38bdc"},"cell_type":"code","source":"# return 1 or 0 - if text includes one or more these characters or not: u'[\\u4e00-\\u9fff]\ndef is_chinese(x):\n    if re.search(u'[\\u4e00-\\u9fff]', x):\n          return 1\n    else: return 0\n\n# no return, but print the sentence if chinese found\ndef is_chinese_print(x):\n    if re.search(u'[\\u4e00-\\u9fff]', x):\n          print('found chinese character in ' + x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"07171e067d1112417ac6f8c1bf6cb0c2232b46b8"},"cell_type":"code","source":"# test it\nis_chinese(('What does 有毒 mean in Chinese?'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e4b85872b7f79506d353df72282f1be9adf3008"},"cell_type":"code","source":"is_chinese_print('What does 有毒 mean in Chinese?')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16e7473155ba59dc438f6c1c85f36f1fedbc3a1e"},"cell_type":"code","source":"is_chinese('english only')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5df53f193795a2e9253d16ac109ac7ddd803f2c7"},"cell_type":"markdown","source":"#### Apply the function to and mark found chinese to new field"},{"metadata":{"trusted":true,"_uuid":"ee72360f99ede15854e77c74d5fe97eb3436a450"},"cell_type":"code","source":"df_train['chinese']= df_train['question_text'].apply(lambda x: is_chinese(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"544c60fd40e57fa8f04780fd2d681a481e1b666e"},"cell_type":"markdown","source":"#### Print chinese"},{"metadata":{"trusted":true,"_uuid":"b6124cf30e4c51dec1ab20764c3b76796c703af0"},"cell_type":"code","source":"df_train[(df_train['chinese']==1)][0:10]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"10cd66b26048c536c22bb119b029f8892868a3a6"},"cell_type":"markdown","source":"#### Count them"},{"metadata":{"trusted":true,"_uuid":"cc7c9b05d5b680f745b04d1738bf843a39f4e224"},"cell_type":"code","source":"df_train['chinese'].sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f4589572e2ab66070fbbcd2f3d2a073a1ca35689"},"cell_type":"markdown","source":"#### Labeled as insincere"},{"metadata":{"trusted":true,"_uuid":"8ec7b04631abe6eee8b869e36e56df6607210245"},"cell_type":"code","source":"df_train[df_train['chinese']==1]['target'].sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3f297f2b3a2ee453f9bd2886fbdfa2612b3dbec8"},"cell_type":"markdown","source":"Only 9 of 292 of questions containing chinese are labeled insincere, that is 0.03 percentage. \nOn whole dataset 2x that many - 0.06 % are insincere."},{"metadata":{"_uuid":"54481819bf3cdbdeade9b1c8e0258073f58d1bbd"},"cell_type":"markdown","source":"#### print the insincere ones"},{"metadata":{"trusted":true,"_uuid":"227c0323c7eb21cbdbef940f96fed0fae06ea25d"},"cell_type":"code","source":"df_train.drop('qid',axis=1)[(df_train['chinese']==1) & (df_train['target']==1)]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"222ec816b70988ea153a98e3219ca53d4e4381ae"},"cell_type":"markdown","source":"We notice that of these 9 labeled insincere, atleast 2 include \"F**k\" and \"Fuck That Bitch\".\n\nFrom others there seems to be some sign of irony about specific subject."}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.8"}},"nbformat":4,"nbformat_minor":1}