{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import os\nimport time\nimport numpy as np\nimport pandas as pd\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport warnings\nwarnings.filterwarnings(\"ignore\")\nimport spacy\nnlp = spacy.load('en')\ntrain = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\nprint(f\"{len(train):,} total training datapoints\")","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.drop('qid', axis=1, inplace=True)\nall_labels = train.pop('target')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# print(train.values[:5])\nprint(all_labels.value_counts())\nprint(all_labels.value_counts(True))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Preprocessing \nimport re\n#Cleaning Special Characters\nNON_CHARACTER = re.compile(r'[^A-Za-z]+')\n#Cleaning Number\nNUMS = re.compile(r'\\d+')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Replace misspelled words using a misspell mapping and regex functions\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.tokenize import word_tokenize\nfrom nltk.stem import WordNetLemmatizer\nl = WordNetLemmatizer()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_sentences(text): \n    text = text.lower().replace('\\\\', '\\\\\\\\')\n    text = NUMS.sub('XXX', text)\n    text = NON_CHARACTER.sub(' ', text)\n    text = replace_typical_misspell(text)\n    # remove endings only if the base form is present in a dictionary\n    text = ' '.join([l.lemmatize(word) for word in word_tokenize(text)])\n    return text \n\nall_texts = np.array([clean_sentences(x[0]) for x in train.values])\nprint(\"Done!\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_x, test_x, train_y, test_y = train_test_split(\n    all_texts, all_labels.values, test_size=0.2, random_state=2019)\nprint(\"Test-train split done!\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = CountVectorizer()\nvectorizer2 = CountVectorizer(min_df=0.0001, max_df=0.999, max_features=5000, ngram_range=(1,2,)) \nbow_train = vectorizer.fit_transform(train_x) \nbow_train2 = vectorizer2.fit_transform(train_x) \nprint(bow_train.shape)\nprint(bow_train2.shape)\nbow_test = vectorizer.transform(test_x)\nbow_test2 = vectorizer2.transform(test_x)\nprint(\"Done creating Bag-of-Words\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.tree import DecisionTreeClassifier","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Results of decision tree on full bag-of-words\")\ndtc = DecisionTreeClassifier(max_depth=30) \ndtc.fit(bow_train, train_y)\ntrain_predictions = dtc.predict(bow_train)\ntrain_acc = accuracy_score(train_y, train_predictions) \ntrain_f1 = f1_score(train_y, train_predictions) \nprint(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \ntest_predictions = dtc.predict(bow_test)\ntest_acc = accuracy_score(test_y, test_predictions) \ntest_f1 = f1_score(test_y, test_predictions) \nprint(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(f\"Results of decision tree on simplified bag-of-words\")\ndtc2 = DecisionTreeClassifier(max_depth=30) \ndtc2.fit(bow_train2, train_y)\ntrain_predictions = dtc2.predict(bow_train2)\ntrain_acc = accuracy_score(train_y, train_predictions) \ntrain_f1 = f1_score(train_y, train_predictions) \nprint(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}, %1: {sum(train_predictions)/len(train_predictions):.2%}\") \ntest_predictions = dtc2.predict(bow_test2)\ntest_acc = accuracy_score(test_y, test_predictions) \ntest_f1 = f1_score(test_y, test_predictions) \nprint(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}, %1: {sum(test_predictions)/len(test_predictions):.2%}\")\n","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}