{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport re\nimport time\nimport gc\nimport random\n\nimport numpy as np\nimport pandas as pd\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":72,"outputs":[{"output_type":"stream","text":"['embeddings', 'sample_submission.csv', 'test.csv', 'train.csv']\n","name":"stdout"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"test_df = pd.read_csv('../input/test.csv')","execution_count":73,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4fd87108ad7a39ccb10e692ed081215aa985f435"},"cell_type":"code","source":"test_df.head()","execution_count":74,"outputs":[{"output_type":"execute_result","execution_count":74,"data":{"text/plain":"                    qid                                      question_text\n0  0000163e3ea7c7a74cd7  Why do so many women become so rude and arroga...\n1  00002bd4fb5d505b9161  When should I apply for RV college of engineer...\n2  00007756b4a147d2b0b3  What is it really like to be a nurse practitio...\n3  000086e4b7e1c7146103                             Who are entrepreneurs?\n4  0000c4c3fbe8785a3090   Is education really making good people nowadays?","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>qid</th>\n      <th>question_text</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>0000163e3ea7c7a74cd7</td>\n      <td>Why do so many women become so rude and arroga...</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>00002bd4fb5d505b9161</td>\n      <td>When should I apply for RV college of engineer...</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>00007756b4a147d2b0b3</td>\n      <td>What is it really like to be a nurse practitio...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>000086e4b7e1c7146103</td>\n      <td>Who are entrepreneurs?</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0000c4c3fbe8785a3090</td>\n      <td>Is education really making good people nowadays?</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"705f088c9ee9c1ff468553f0fe491d7b871f3cd7"},"cell_type":"code","source":"train_df = pd.read_csv('../input/train.csv')","execution_count":75,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"200bccacdde39cbdceec05fbca2add295795ed8f"},"cell_type":"code","source":"train_df.head()","execution_count":76,"outputs":[{"output_type":"execute_result","execution_count":76,"data":{"text/plain":"                    qid  ...   target\n0  00002165364db923c7e6  ...        0\n1  000032939017120e6e44  ...        0\n2  0000412ca6e4628ce2cf  ...        0\n3  000042bf85aa498cd78e  ...        0\n4  0000455dfa3e01eae3af  ...        0\n\n[5 rows x 3 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>qid</th>\n      <th>question_text</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>00002165364db923c7e6</td>\n      <td>How did Quebec nationalists see their province...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>000032939017120e6e44</td>\n      <td>Do you have an adopted dog, how would you enco...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>0000412ca6e4628ce2cf</td>\n      <td>Why does velocity affect time? Does velocity a...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>000042bf85aa498cd78e</td>\n      <td>How did Otto von Guericke used the Magdeburg h...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>0000455dfa3e01eae3af</td>\n      <td>Can I convert montra helicon D to a mountain b...</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"537aea0554b7124905d438a7f2aa8d34e81d05db"},"cell_type":"code","source":"train_df.describe()","execution_count":77,"outputs":[{"output_type":"execute_result","execution_count":77,"data":{"text/plain":"             target\ncount  1.306122e+06\nmean   6.187018e-02\nstd    2.409197e-01\nmin    0.000000e+00\n25%    0.000000e+00\n50%    0.000000e+00\n75%    0.000000e+00\nmax    1.000000e+00","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>count</th>\n      <td>1.306122e+06</td>\n    </tr>\n    <tr>\n      <th>mean</th>\n      <td>6.187018e-02</td>\n    </tr>\n    <tr>\n      <th>std</th>\n      <td>2.409197e-01</td>\n    </tr>\n    <tr>\n      <th>min</th>\n      <td>0.000000e+00</td>\n    </tr>\n    <tr>\n      <th>25%</th>\n      <td>0.000000e+00</td>\n    </tr>\n    <tr>\n      <th>50%</th>\n      <td>0.000000e+00</td>\n    </tr>\n    <tr>\n      <th>75%</th>\n      <td>0.000000e+00</td>\n    </tr>\n    <tr>\n      <th>max</th>\n      <td>1.000000e+00</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"edacba18cd405e8b99e8511945bf367618b569f2"},"cell_type":"code","source":"train_df.info()","execution_count":78,"outputs":[{"output_type":"stream","text":"<class 'pandas.core.frame.DataFrame'>\nRangeIndex: 1306122 entries, 0 to 1306121\nData columns (total 3 columns):\nqid              1306122 non-null object\nquestion_text    1306122 non-null object\ntarget           1306122 non-null int64\ndtypes: int64(1), object(2)\nmemory usage: 29.9+ MB\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"c1b85b54747356679115048ed841eef90b0e4d8c"},"cell_type":"code","source":"train_df.groupby('target').qid.count()","execution_count":79,"outputs":[{"output_type":"execute_result","execution_count":79,"data":{"text/plain":"target\n0    1225312\n1      80810\nName: qid, dtype: int64"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"20613e9a388dd9e8013ec27ec4e0267bbd8c60ad"},"cell_type":"code","source":"80810/1306122\n","execution_count":80,"outputs":[{"output_type":"execute_result","execution_count":80,"data":{"text/plain":"0.06187017751787352"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"2e6b3f87aaef1d8dbf5bf6ed9bf21055f6032d64"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b055fe587d12c4c93836a9e576910561933a5b48"},"cell_type":"code","source":"\n\nX_train, X_test, y_train, y_test = train_test_split(train_df['question_text'], \n                                                    train_df['target'], \n                                                    random_state=1)\n\nprint('Number of rows in the total set: {}'.format(train_df.shape[0]))\nprint('Number of rows in the training set: {}'.format(X_train.shape[0]))\nprint('Number of rows in the test set: {}'.format(X_test.shape[0]))","execution_count":81,"outputs":[{"output_type":"stream","text":"Number of rows in the total set: 1306122\nNumber of rows in the training set: 979591\nNumber of rows in the test set: 326531\n","name":"stdout"}]},{"metadata":{"trusted":true,"_uuid":"425dac699474bb37c54a2218d9cbd7cbb45af831"},"cell_type":"code","source":"# Instantiate the CountVectorizer method\ncount_vector = CountVectorizer()\n\n# Fit the training data and then return the matrix\ntraining_data = count_vector.fit_transform(X_train)\n\n# Transform testing data and return the matrix. Note we are not fitting the testing data into the CountVectorizer()\ntesting_data = count_vector.transform(X_test)","execution_count":82,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"050c98b4a08b3a01f8f692889e84d5e3f2949cf1"},"cell_type":"code","source":"naive_bayes = MultinomialNB()\nnaive_bayes.fit(training_data, y_train)","execution_count":83,"outputs":[{"output_type":"execute_result","execution_count":83,"data":{"text/plain":"MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"},"metadata":{}}]},{"metadata":{"trusted":true,"_uuid":"ef9d31813cfb190289f3fa8015efca82de541a7d"},"cell_type":"code","source":"predictions = naive_bayes.predict(testing_data)","execution_count":84,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"390e38bd3b88dfced2609953032a7650b760ed84"},"cell_type":"code","source":"print('Accuracy score: ', format(accuracy_score(y_test, predictions)))\nprint('Precision score: ', format(precision_score(y_test, predictions)))\nprint('Recall score: ', format(recall_score(y_test, predictions)))\nprint('F1 score: ', format(f1_score(y_test, predictions)))","execution_count":85,"outputs":[{"output_type":"stream","text":"Accuracy score:  0.9336357038076017\nPrecision score:  0.47771268030287456\nRecall score:  0.6845205950218469\nF1 score:  0.5627169263055936\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"solution_data = count_vector.transform(test_df['question_text'])\nTest_predictions = naive_bayes.predict(solution_data)","execution_count":86,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df = pd.DataFrame(np.array(Test_predictions),columns = ['prediction'] ,index = test_df['qid'])","execution_count":87,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.to_csv('submission.csv', index = True)","execution_count":88,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.head()","execution_count":89,"outputs":[{"output_type":"execute_result","execution_count":89,"data":{"text/plain":"                      prediction\nqid                             \n0000163e3ea7c7a74cd7           1\n00002bd4fb5d505b9161           0\n00007756b4a147d2b0b3           0\n000086e4b7e1c7146103           0\n0000c4c3fbe8785a3090           0","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>prediction</th>\n    </tr>\n    <tr>\n      <th>qid</th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0000163e3ea7c7a74cd7</th>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>00002bd4fb5d505b9161</th>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>00007756b4a147d2b0b3</th>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>000086e4b7e1c7146103</th>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>0000c4c3fbe8785a3090</th>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}