{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import pandas as pd\nimport string\nimport re\nimport numpy as np\nfrom collections import Counter\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.ensemble import AdaBoostClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.svm import SVC\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn import model_selection\nfrom sklearn.metrics import classification_report\nfrom sklearn import preprocessing\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.multiclass import OneVsRestClassifier\nfrom sklearn.metrics import cohen_kappa_score, make_scorer\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer\n\nstop_words = set(stopwords.words('english'))\n\ntrain = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\nsample_submission = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-10-16T00:57:53.593346Z","iopub.execute_input":"2021-10-16T00:57:53.593945Z","iopub.status.idle":"2021-10-16T00:58:01.825107Z","shell.execute_reply.started":"2021-10-16T00:57:53.593802Z","shell.execute_reply":"2021-10-16T00:58:01.824385Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#base desbalanceada, precisamos corrigir\n\ntrain_balanced = train[train['target'] == 1]\ntrain_balanced = train_balanced.append(train[train['target'] == 0].sample(n = len(train_balanced))).reset_index(drop = True)\ntrain_balanced['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-10-16T00:58:22.635855Z","iopub.execute_input":"2021-10-16T00:58:22.636302Z","iopub.status.idle":"2021-10-16T00:58:22.884729Z","shell.execute_reply.started":"2021-10-16T00:58:22.636270Z","shell.execute_reply":"2021-10-16T00:58:22.883847Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean(sen):\n    text = sen.lower()\n  \n    tokenized_word = word_tokenize(text)\n    ps = PorterStemmer()\n    stemmed_words=[]\n    for w in tokenized_word:\n        stemmed_words.append(ps.stem(w))\n\n    text = ' '.join(stemmed_words)\n\n    return text\n\ntrain_text = train_balanced['question_text'].apply(lambda text: clean(text))\ntrain_target = train_balanced['target'].values\n\ntest_text = test['question_text'].apply(lambda text: clean(text))","metadata":{"execution":{"iopub.status.busy":"2021-10-16T00:58:27.147199Z","iopub.execute_input":"2021-10-16T00:58:27.147509Z","iopub.status.idle":"2021-10-16T01:02:42.235297Z","shell.execute_reply.started":"2021-10-16T00:58:27.147473Z","shell.execute_reply":"2021-10-16T01:02:42.234151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = TfidfVectorizer(use_idf=True, ngram_range=(1,2))\ntfidf_model = vectorizer.fit(train_text)\ntrain_tfidf = tfidf_model.transform(train_text)\ntest_tfidf = tfidf_model.transform(test_text)","metadata":{"execution":{"iopub.status.busy":"2021-10-16T01:03:41.712865Z","iopub.execute_input":"2021-10-16T01:03:41.713558Z","iopub.status.idle":"2021-10-16T01:04:02.562524Z","shell.execute_reply.started":"2021-10-16T01:03:41.713505Z","shell.execute_reply":"2021-10-16T01:04:02.561583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = MultinomialNB()\nclf = model.fit(train_tfidf, train_target)","metadata":{"execution":{"iopub.status.busy":"2021-10-16T01:04:06.451237Z","iopub.execute_input":"2021-10-16T01:04:06.451987Z","iopub.status.idle":"2021-10-16T01:04:06.563616Z","shell.execute_reply.started":"2021-10-16T01:04:06.451935Z","shell.execute_reply":"2021-10-16T01:04:06.562659Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = clf.predict(test_tfidf)","metadata":{"execution":{"iopub.status.busy":"2021-10-16T01:04:09.160438Z","iopub.execute_input":"2021-10-16T01:04:09.161415Z","iopub.status.idle":"2021-10-16T01:04:09.232933Z","shell.execute_reply.started":"2021-10-16T01:04:09.161365Z","shell.execute_reply":"2021-10-16T01:04:09.232001Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"targets = pd.Series(targets)\nsubm_df = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')\nsubm_df['prediction'] = targets\nsubm_df.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-10-16T01:04:11.203078Z","iopub.execute_input":"2021-10-16T01:04:11.203368Z","iopub.status.idle":"2021-10-16T01:04:12.430914Z","shell.execute_reply.started":"2021-10-16T01:04:11.203338Z","shell.execute_reply":"2021-10-16T01:04:12.429952Z"},"trusted":true},"execution_count":null,"outputs":[]}]}