{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-12-06T13:54:14.463556Z","iopub.execute_input":"2022-12-06T13:54:14.464470Z","iopub.status.idle":"2022-12-06T13:54:14.491277Z","shell.execute_reply.started":"2022-12-06T13:54:14.464364Z","shell.execute_reply":"2022-12-06T13:54:14.490321Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import os\nimport pandas as pd\nfrom pathlib import Path","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:14.499310Z","iopub.execute_input":"2022-12-06T13:54:14.499635Z","iopub.status.idle":"2022-12-06T13:54:14.504350Z","shell.execute_reply.started":"2022-12-06T13:54:14.499605Z","shell.execute_reply":"2022-12-06T13:54:14.503197Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_dir = Path(\"/kaggle/input/quora-insincere-questions-classification\")","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:14.506273Z","iopub.execute_input":"2022-12-06T13:54:14.507003Z","iopub.status.idle":"2022-12-06T13:54:14.513977Z","shell.execute_reply.started":"2022-12-06T13:54:14.506968Z","shell.execute_reply":"2022-12-06T13:54:14.513017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"os.listdir(data_dir)\n\nraw_train = pd.read_csv(data_dir/\"train.csv\")\n\nraw_train.head()\n\n#Insinsere questions\nraw_train[raw_train.target==1].sample(30).question_text.values\n\nraw_train.target.value_counts(normalize=True).plot(kind=\"bar\")\n\nraw_train.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:14.515585Z","iopub.execute_input":"2022-12-06T13:54:14.516325Z","iopub.status.idle":"2022-12-06T13:54:19.533282Z","shell.execute_reply.started":"2022-12-06T13:54:14.516271Z","shell.execute_reply":"2022-12-06T13:54:19.532240Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sample_df = raw_train","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:19.536135Z","iopub.execute_input":"2022-12-06T13:54:19.537131Z","iopub.status.idle":"2022-12-06T13:54:19.542347Z","shell.execute_reply.started":"2022-12-06T13:54:19.537090Z","shell.execute_reply":"2022-12-06T13:54:19.541032Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## NLTK LIBRARY","metadata":{}},{"cell_type":"code","source":"import nltk\n\nfrom nltk.tokenize import word_tokenize\nnltk.download('punkt')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:19.543942Z","iopub.execute_input":"2022-12-06T13:54:19.544877Z","iopub.status.idle":"2022-12-06T13:54:20.815898Z","shell.execute_reply.started":"2022-12-06T13:54:19.544772Z","shell.execute_reply":"2022-12-06T13:54:20.814845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.corpus import stopwords\nnltk.download('stopwords')\neng_Stop = stopwords.words('english')","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:20.818316Z","iopub.execute_input":"2022-12-06T13:54:20.818908Z","iopub.status.idle":"2022-12-06T13:54:20.832160Z","shell.execute_reply.started":"2022-12-06T13:54:20.818870Z","shell.execute_reply":"2022-12-06T13:54:20.831016Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eng_Stop","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:20.833787Z","iopub.execute_input":"2022-12-06T13:54:20.834133Z","iopub.status.idle":"2022-12-06T13:54:20.842793Z","shell.execute_reply.started":"2022-12-06T13:54:20.834093Z","shell.execute_reply":"2022-12-06T13:54:20.841882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_stopwords(tokens):\n      return [token.lower() for token in tokens if token.lower() not in eng_Stop]","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:20.844219Z","iopub.execute_input":"2022-12-06T13:54:20.844855Z","iopub.status.idle":"2022-12-06T13:54:20.853618Z","shell.execute_reply.started":"2022-12-06T13:54:20.844822Z","shell.execute_reply":"2022-12-06T13:54:20.852562Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.stem.snowball import SnowballStemmer\n\nstem = SnowballStemmer(language=\"english\")","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:20.858895Z","iopub.execute_input":"2022-12-06T13:54:20.859164Z","iopub.status.idle":"2022-12-06T13:54:20.865012Z","shell.execute_reply.started":"2022-12-06T13:54:20.859141Z","shell.execute_reply":"2022-12-06T13:54:20.863993Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Define the Vecorizer","metadata":{}},{"cell_type":"code","source":"from sklearn.feature_extraction.text import CountVectorizer","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:20.866520Z","iopub.execute_input":"2022-12-06T13:54:20.866959Z","iopub.status.idle":"2022-12-06T13:54:20.873604Z","shell.execute_reply.started":"2022-12-06T13:54:20.866923Z","shell.execute_reply":"2022-12-06T13:54:20.872344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def tokenize(text):\n      return [stem.stem(tokens) for tokens in word_tokenize(text)]\n\nvectorizer = CountVectorizer(lowercase=True,\n                             tokenizer = tokenize,\n                             stop_words = eng_Stop,\n                             max_features = 50000)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:20.875369Z","iopub.execute_input":"2022-12-06T13:54:20.875770Z","iopub.status.idle":"2022-12-06T13:54:20.885128Z","shell.execute_reply.started":"2022-12-06T13:54:20.875737Z","shell.execute_reply":"2022-12-06T13:54:20.884186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\ninputs = vectorizer.fit_transform(sample_df.question_text)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T13:54:20.887877Z","iopub.execute_input":"2022-12-06T13:54:20.888198Z","iopub.status.idle":"2022-12-06T14:00:42.691728Z","shell.execute_reply.started":"2022-12-06T13:54:20.888152Z","shell.execute_reply":"2022-12-06T14:00:42.690671Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels = sample_df.target","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:00:42.693219Z","iopub.execute_input":"2022-12-06T14:00:42.696129Z","iopub.status.idle":"2022-12-06T14:00:42.701878Z","shell.execute_reply.started":"2022-12-06T14:00:42.696098Z","shell.execute_reply":"2022-12-06T14:00:42.700942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"labels.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:00:42.704684Z","iopub.execute_input":"2022-12-06T14:00:42.704986Z","iopub.status.idle":"2022-12-06T14:00:42.713201Z","shell.execute_reply.started":"2022-12-06T14:00:42.704959Z","shell.execute_reply":"2022-12-06T14:00:42.712577Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training Logistic Model","metadata":{}},{"cell_type":"code","source":"import sklearn\n\nX_train, X_val, y_train, y_val = sklearn.model_selection.train_test_split( inputs, labels, test_size=0.2, random_state=42)\n\nX_train.shape\n\ny_train.shape\n\nX_val.shape","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:00:42.714025Z","iopub.execute_input":"2022-12-06T14:00:42.714263Z","iopub.status.idle":"2022-12-06T14:00:42.938229Z","shell.execute_reply.started":"2022-12-06T14:00:42.714240Z","shell.execute_reply":"2022-12-06T14:00:42.937201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\n%%time\ntest_data = pd.read_csv(data_dir/'test.csv')\n\ntest_data\n\ntest_data = vectorizer.transform(test_data.question_text.values)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:01:25.968020Z","iopub.execute_input":"2022-12-06T14:01:25.968389Z","iopub.status.idle":"2022-12-06T14:03:16.035767Z","shell.execute_reply.started":"2022-12-06T14:01:25.968358Z","shell.execute_reply":"2022-12-06T14:03:16.034689Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nlreg = sklearn.linear_model.LogisticRegression(solver=\"sag\", max_iter=1000)\n\nlreg.fit(X_train, y_train)","metadata":{"execution":{"iopub.status.busy":"2022-12-06T14:59:20.008740Z","iopub.execute_input":"2022-12-06T14:59:20.009101Z","iopub.status.idle":"2022-12-06T15:25:07.677122Z","shell.execute_reply.started":"2022-12-06T14:59:20.009070Z","shell.execute_reply":"2022-12-06T15:25:07.676170Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_hat = lreg.predict(X_val)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score as accuracy\nfrom sklearn.metrics import f1_score\n\n\nprint(\"accuracy = \",accuracy(y_hat, y_val))\n\nprint(\"f1 = \", f1_score(y_hat, y_val))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_predictions = lreg.predict(test_data)\n\npd.Series(test_predictions).value_counts()\n\nsubmission = pd.read_csv(data_dir/\"sample_submission.csv\")\n\nsubmission\n\nsubmission.prediction = test_predictions\n\nsubmission","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"submission.to_csv(\"submission.csv\", index=None)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}