{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"scrolled":true},"cell_type":"code","source":"df = pd.read_csv('../input/train.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"323be5bfdf6a73920f2dddbefc6e35957da33bc7"},"cell_type":"code","source":"df['target'].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b067c94a1a89074f908ebd860eda5fc050dc4faa"},"cell_type":"markdown","source":"### Resampling imbalanced dataset\n\nTrying undersampling strategy:"},{"metadata":{"trusted":true,"_uuid":"c314e273d756f28ad11c0c7501252014495657ef"},"cell_type":"code","source":"count_target_0, count_target_1 = df['target'].value_counts()\n\ndf_target_0 = df[df['target'] == 0]\ndf_target_1 = df[df['target'] == 1]\n\ndf_target_0_under = df_target_0.sample(count_target_1)\ndf_under = pd.concat([df_target_0_under, df_target_1], axis=0)\n\ndf_under['target'].value_counts().plot(kind='bar', title='Count (target)')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"00c3a7b82b1e53845e2a656dec1d833fe70770ec"},"cell_type":"markdown","source":"### Applying"},{"metadata":{"trusted":true,"_uuid":"1c5216d7038b6a6fc7cdd7d0194cf4cf8f5e949b"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import TfidfVectorizer\n\n\nX_train, X_test, y_train, y_test = train_test_split(df_under['question_text'],\n                                                    df_under['target'],\n                                                    test_size=0.2)\ntf_vectorizer = TfidfVectorizer().fit(df_under['question_text'])\nX_train = tf_vectorizer.transform(X_train)\nX_test = tf_vectorizer.transform(X_test)\nX_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"04a680bab8f3a8fe2ad8802b1d2761bf03987be9"},"cell_type":"code","source":"from sklearn.naive_bayes import MultinomialNB\n\n\nclf = MultinomialNB().fit(X_train, y_train)\npredicted = clf.predict(X_test)\nnp.mean(predicted == y_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd65d4ac8c6a4a35c95cd78687345601af4107a6"},"cell_type":"code","source":"from sklearn.metrics import f1_score\n\n\nf1_score(y_test, predicted,average=None)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"80790e9798092407b93d62547d568ca34d1e2e36"},"cell_type":"markdown","source":"### Submission dataset"},{"metadata":{"trusted":true,"_uuid":"6d2a83008b4a507d9c606a5e0f120b00c0d2caf4"},"cell_type":"code","source":"df_test = pd.read_csv('../input/test.csv')\nX_submission = tf_vectorizer.transform(df_test['question_text'])\npredicted_test = clf.predict(X_submission)\n\ndf_test['prediction'] = predicted_test\nsubmission = df_test.drop(columns=['question_text'])\nsubmission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50fba64d48f018a5db8f80f1e0657aca58848005"},"cell_type":"code","source":"submission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}