{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"## Quora Incinsere Questions Classification via Logistic Regression","metadata":{}},{"cell_type":"code","source":"# Import required library\nimport os\nimport time\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt \nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n# import spacy\nimport re\nfrom tqdm import tqdm\nimport nltk\nfrom tqdm import tqdm_notebook\ntqdm_notebook().pandas()\n# nltk.download('punkt')\n# nltk.download('wordnet')","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-10T19:17:42.484764Z","iopub.execute_input":"2021-06-10T19:17:42.485383Z","iopub.status.idle":"2021-06-10T19:17:42.524107Z","shell.execute_reply.started":"2021-06-10T19:17:42.485320Z","shell.execute_reply":"2021-06-10T19:17:42.523176Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Load and print data","metadata":{}},{"cell_type":"code","source":"train_raw = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\nvalidation_data = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\ntrain_raw","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:17:42.525616Z","iopub.execute_input":"2021-06-10T19:17:42.525905Z","iopub.status.idle":"2021-06-10T19:17:46.420658Z","shell.execute_reply.started":"2021-06-10T19:17:42.525878Z","shell.execute_reply":"2021-06-10T19:17:46.419569Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data\n- 1306122 row × 3 column\n- Features include question ID, question, target of the question\n- No percentage between different types of target","metadata":{}},{"cell_type":"code","source":"plot_data = train_raw['target']\nplot_data.value_counts().plot(kind='bar', rot=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:17:46.422358Z","iopub.execute_input":"2021-06-10T19:17:46.422628Z","iopub.status.idle":"2021-06-10T19:17:46.583942Z","shell.execute_reply.started":"2021-06-10T19:17:46.422602Z","shell.execute_reply":"2021-06-10T19:17:46.582894Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Pre process raw data, add features to observe and utilize\n- Number of words\n- Number of unique words\n- Number of special characters\n- Number of upper-case words\n- Number of lower-case words\n- Number of tittle-case words","metadata":{}},{"cell_type":"code","source":"def create_features(df_):\n    \n    df_[\"nb_words\"] = df_[\"question_text\"].apply(lambda x: len(x.split())) # Number of words\n    df_[\"nb_unique_words\"] = df_[\"question_text\"].apply(lambda x: len(set(str(x).split()))) # Number of unique words\n    df_[\"nb_chars\"] = df_[\"question_text\"].apply(lambda x: len(str(x))) # Number of characters\n    df_['spe_chars'] = df_['question_text'].str.findall(r'[^a-zA-Z0-9 ]').str.len() # Number of special characters\n    df_[\"nb_uppercase\"] = df_[\"question_text\"].apply(lambda x : len([nu for nu in str(x).split() if nu.isupper()])) # Number of uppercase characters\n    df_[\"nb_lowercase\"] = df_[\"question_text\"].apply(lambda x : len([nl for nl in str(x).split() if nl.islower()])) # Number of lowercase characters\n    df_[\"nb_title\"] = df_[\"question_text\"].apply(lambda x : len([nl for nl in str(x).split() if nl.istitle()])) # Number of tittle case\n\n    return df_\n\ntrain_features = create_features(train_raw)\ntrain_features","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:17:46.585598Z","iopub.execute_input":"2021-06-10T19:17:46.585903Z","iopub.status.idle":"2021-06-10T19:18:10.800613Z","shell.execute_reply.started":"2021-06-10T19:17:46.585874Z","shell.execute_reply":"2021-06-10T19:18:10.799826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sincere questions data","metadata":{}},{"cell_type":"code","source":"train_features[train_features['target'] == 0].describe().round(1)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:10.801724Z","iopub.execute_input":"2021-06-10T19:18:10.802107Z","iopub.status.idle":"2021-06-10T19:18:11.325868Z","shell.execute_reply.started":"2021-06-10T19:18:10.802078Z","shell.execute_reply":"2021-06-10T19:18:11.324874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Insincere questions data","metadata":{}},{"cell_type":"code","source":"train_features[train_features['target'] == 1].describe().round(1)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:11.327247Z","iopub.execute_input":"2021-06-10T19:18:11.327543Z","iopub.status.idle":"2021-06-10T19:18:11.415806Z","shell.execute_reply.started":"2021-06-10T19:18:11.327516Z","shell.execute_reply":"2021-06-10T19:18:11.415024Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observation:\n- Insincere questions tends to have larger mean in the number of words than sincere questions\n- Word vector can be used (word vectors are capable of holding features such as number or words, number of unique words..)","metadata":{}},{"cell_type":"code","source":"## Câu hỏi insincere\nprint(train_raw['question_text'][(train_raw['target']==1)].sample(10).values)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:11.417709Z","iopub.execute_input":"2021-06-10T19:18:11.418103Z","iopub.status.idle":"2021-06-10T19:18:11.440790Z","shell.execute_reply.started":"2021-06-10T19:18:11.418073Z","shell.execute_reply":"2021-06-10T19:18:11.439759Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Pre-processing evaluation:\n- Insincered question often has negative meaning words, this feature is not dependent on grammar\n#### => Word counting vector can be used since grammar of the question can be discarded\n- Special characters, numbers, links, uppercase or lowercase letter often don't affect the classification of the questions\n#### => They can be discarded","metadata":{}},{"cell_type":"markdown","source":"### Cleaning data from abnormal cases\n- From the statistics table, observe that some questions has the number of characters, special characters larger than the mean of the whole dataset\n#### => Consider remove these cases","metadata":{}},{"cell_type":"code","source":"train_features['question_text'][(train_raw['target']==1) & (train_features['nb_chars']>600.0)].values\ntrain_features['question_text'][(train_raw['target']==1) & (train_features['spe_chars']>30.0)].values","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:11.442969Z","iopub.execute_input":"2021-06-10T19:18:11.443410Z","iopub.status.idle":"2021-06-10T19:18:11.465255Z","shell.execute_reply.started":"2021-06-10T19:18:11.443377Z","shell.execute_reply":"2021-06-10T19:18:11.464457Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_features['question_text'][(train_raw['target']==0) & (train_features['nb_chars']>600.0)].values\ntrain_features['question_text'][(train_raw['target']==0) & (train_features['spe_chars']>30.0)].values","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:11.466621Z","iopub.execute_input":"2021-06-10T19:18:11.467105Z","iopub.status.idle":"2021-06-10T19:18:11.492985Z","shell.execute_reply.started":"2021-06-10T19:18:11.467074Z","shell.execute_reply":"2021-06-10T19:18:11.492223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observation:\n- Most questions that has a lot of special characters are questions with mathematical formular or figurative characters\n- For questions with mathematical formular that is classified as insincere, they can be considered anomaly and can be removed from the dataset","metadata":{}},{"cell_type":"markdown","source":"### Remove questions which has number of characters, special characters exceeding threshold","metadata":{}},{"cell_type":"code","source":"# train_features_filtered = train_features.drop(train_features[(train_features['nb_chars'] >= 600) & (train_features['nb_unique_words']>35.0)].index)\ntrain_features_filtered = train_features[(train_features['nb_chars']<600.0) & (train_features['nb_words']<70.0) & (train_features['spe_chars']<12.0)]\ntrain_features_filtered.describe().round(1)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:11.494219Z","iopub.execute_input":"2021-06-10T19:18:11.494752Z","iopub.status.idle":"2021-06-10T19:18:12.052224Z","shell.execute_reply.started":"2021-06-10T19:18:11.494719Z","shell.execute_reply":"2021-06-10T19:18:12.051114Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(train_features) - len(train_features_filtered))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:12.053501Z","iopub.execute_input":"2021-06-10T19:18:12.053807Z","iopub.status.idle":"2021-06-10T19:18:12.057984Z","shell.execute_reply.started":"2021-06-10T19:18:12.053777Z","shell.execute_reply":"2021-06-10T19:18:12.057335Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"#### 3494 rows removed","metadata":{}},{"cell_type":"markdown","source":"## Resample\n- The number of sincere questions is far greater than the number of insincere questions, the ratio is not balanced\n#### => The dataset needs to be resampled to the ratio of 4:1","metadata":{}},{"cell_type":"code","source":"# Resampling\nfrom sklearn.utils import resample\n\n# sincere = train_raw[train_raw.target == 0]\n# insincere = train_raw[train_raw.target == 1]\n\nsincere = train_features_filtered[train_features_filtered.target == 0]\ninsincere = train_features_filtered[train_features_filtered.target == 1]\n\n# ratio 1:1\n# x = pd.concat([resample(sincere,\n#                      replace = False,\n#                      n_samples = len(insincere)), insincere])\n\n# ratio 2:1\n# x = pd.concat([resample(sincere,\n#                      replace = True,\n#                      n_samples = len(insincere)*2), insincere])\n\n# 4:1\nresampled_dataset = pd.concat([resample(sincere,\n                     replace = True,\n                     n_samples = len(insincere)*4), insincere])\nprint(len(resampled_dataset[resampled_dataset['target'] == 0]) / len(resampled_dataset[resampled_dataset['target'] == 1]))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:12.059002Z","iopub.execute_input":"2021-06-10T19:18:12.059465Z","iopub.status.idle":"2021-06-10T19:18:12.746451Z","shell.execute_reply.started":"2021-06-10T19:18:12.059428Z","shell.execute_reply":"2021-06-10T19:18:12.745458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_data = resampled_dataset['target']\nplot_data.value_counts().plot(kind='bar', rot=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:12.747758Z","iopub.execute_input":"2021-06-10T19:18:12.748055Z","iopub.status.idle":"2021-06-10T19:18:12.891642Z","shell.execute_reply.started":"2021-06-10T19:18:12.748024Z","shell.execute_reply":"2021-06-10T19:18:12.890595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Training dataset is now balanced","metadata":{}},{"cell_type":"markdown","source":"## Process data\n### Loại bỏ các dữ liệu không cần thiết và chuyển dữ liệu về dạng nguyên gốc:\n- Remove links\n- Remove special characters\n- Transform variants of one word into one consistent word\n- Transform shortened word into original word\n- Remove numbers\n- Remove latex tags","metadata":{}},{"cell_type":"code","source":"# Remove links\ndef clean_tag(question):\n    if 'http' in question or 'www' in question:\n        question = re.sub('(?:(?:https?|ftp):\\/\\/)?[\\w/\\-?=%.]+\\.[\\w/\\-?=%.]+', '[url]', question) #replacing with [url]\n    return question","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:12.892938Z","iopub.execute_input":"2021-06-10T19:18:12.893231Z","iopub.status.idle":"2021-06-10T19:18:12.897934Z","shell.execute_reply.started":"2021-06-10T19:18:12.893203Z","shell.execute_reply":"2021-06-10T19:18:12.896871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Transform shortened into original\n\ncontraction_mapping = {\"We'd\": \"We had\", \"That'd\": \"That had\", \"AREN'T\": \"Are not\", \"HADN'T\": \"Had not\", \"Could've\": \"Could have\", \"LeT's\": \"Let us\", \"How'll\": \"How will\", \"They'll\": \"They will\", \"DOESN'T\": \"Does not\", \"HE'S\": \"He has\", \"O'Clock\": \"Of the clock\", \"Who'll\": \"Who will\", \"What'S\": \"What is\", \"Ain't\": \"Am not\", \"WEREN'T\": \"Were not\", \"Y'all\": \"You all\", \"Y'ALL\": \"You all\", \"Here's\": \"Here is\", \"It'd\": \"It had\", \"Should've\": \"Should have\", \"I'M\": \"I am\", \"ISN'T\": \"Is not\", \"Would've\": \"Would have\", \"He'll\": \"He will\", \"DON'T\": \"Do not\", \"She'd\": \"She had\", \"WOULDN'T\": \"Would not\", \"She'll\": \"She will\", \"IT's\": \"It is\", \"There'd\": \"There had\", \"It'll\": \"It will\", \"You'll\": \"You will\", \"He'd\": \"He had\", \"What'll\": \"What will\", \"Ma'am\": \"Madam\", \"CAN'T\": \"Can not\", \"THAT'S\": \"That is\", \"You've\": \"You have\", \"She's\": \"She is\", \"Weren't\": \"Were not\", \"They've\": \"They have\", \"Couldn't\": \"Could not\", \"When's\": \"When is\", \"Haven't\": \"Have not\", \"We'll\": \"We will\", \"That's\": \"That is\", \"We're\": \"We are\", \"They're\": \"They' are\", \"You'd\": \"You would\", \"How'd\": \"How did\", \"What're\": \"What are\", \"Hasn't\": \"Has not\", \"Wasn't\": \"Was not\", \"Won't\": \"Will not\", \"There's\": \"There is\", \"Didn't\": \"Did not\", \"Doesn't\": \"Does not\", \"You're\": \"You are\", \"He's\": \"He is\", \"SO's\": \"So is\", \"We've\": \"We have\", \"Who's\": \"Who is\", \"Wouldn't\": \"Would not\", \"Why's\": \"Why is\", \"WHO's\": \"Who is\", \"Let's\": \"Let us\", \"How's\": \"How is\", \"Can't\": \"Can not\", \"Where's\": \"Where is\", \"They'd\": \"They had\", \"Don't\": \"Do not\", \"Shouldn't\":\"Should not\", \"Aren't\":\"Are not\", \"ain't\": \"is not\", \"What's\": \"What is\", \"It's\": \"It is\", \"Isn't\":\"Is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\n\ndef clean_contractions(question):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        question = question.replace(s, \"'\")\n    \n    question = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in question.split(\" \")])\n    return question","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:12.899669Z","iopub.execute_input":"2021-06-10T19:18:12.899984Z","iopub.status.idle":"2021-06-10T19:18:12.922001Z","shell.execute_reply.started":"2021-06-10T19:18:12.899953Z","shell.execute_reply":"2021-06-10T19:18:12.921146Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tranform variants of words in to one consistent variants\nfrom nltk.tokenize import word_tokenize\nfrom nltk.stem import WordNetLemmatizer\nl = WordNetLemmatizer()\n\ndef lemmatize_text(question):\n    question = ' '.join([l.lemmatize(word) for word in word_tokenize(question)])\n    return question","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:12.922966Z","iopub.execute_input":"2021-06-10T19:18:12.923227Z","iopub.status.idle":"2021-06-10T19:18:12.940959Z","shell.execute_reply.started":"2021-06-10T19:18:12.923201Z","shell.execute_reply":"2021-06-10T19:18:12.939965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove special characters\nspec_chars = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', \n        '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', \n        '█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', \n        '—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞', \n        '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━', \n        '⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○', \n        '◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐', \n        '├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\\x97', '↺', \n        '∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊', \n        '┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´', \n        '↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\\x93', '≧', '］', \n        '\\x94', '∀', '♛', '\\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％', \n        '╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭', \n        '┊', '╯', '☾', '➔', '∴', '\\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐', \n        '▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\\x91', '➚', '◡', '╰', '\\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒', \n        '☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜', \n        '‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢', \n        '≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥', \n        '⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ', \n        '✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦', \n        '◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ', \n        '⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋', \n        '､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞', \n        '◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ', \n        '✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱', \n        '☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹', \n        '⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚', \n        '☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛', \n        '⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙', \n        'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘', \n        '˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛', \n        '☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑', \n        '✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ', \n        '⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘', \n        '⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊', \n        '⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷', \n        '◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝', \n        '⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢', \n        '≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']\n\ndef clean_spec_chars(question):\n  for spec_char in spec_chars:\n    if spec_char in question:\n      question = question.replace(spec_char, f' {spec_char} ')\n  return question","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:12.942629Z","iopub.execute_input":"2021-06-10T19:18:12.942956Z","iopub.status.idle":"2021-06-10T19:18:12.981332Z","shell.execute_reply.started":"2021-06-10T19:18:12.942925Z","shell.execute_reply":"2021-06-10T19:18:12.980334Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove numeric characters\ndef clean_numbers(question):\n    question = re.sub(r'(\\d+)([a-zA-Z])', '\\g<1> \\g<2>', question)\n    question = re.sub(r'(\\d+) (th|st|nd|rd) ', '\\g<1>\\g<2> ', question)\n    question = re.sub(r'(\\d+),(\\d+)', '\\g<1>\\g<2>', question)\n    return question","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:12.982727Z","iopub.execute_input":"2021-06-10T19:18:12.983028Z","iopub.status.idle":"2021-06-10T19:18:13.000710Z","shell.execute_reply.started":"2021-06-10T19:18:12.983000Z","shell.execute_reply":"2021-06-10T19:18:12.999483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove latext formula\ndef clean_latex_formula(question):\n    corr_t = []\n    for t in question.split(\" \"):\n        t = question.strip()\n        if t != '':\n            corr_t.append(t)\n    question = ' '.join(corr_t)\n    question = re.sub('(\\[ math \\]).+(\\[ / math \\])', 'math formula', question) # replace with \"math formula\"\n    return question","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:13.005196Z","iopub.execute_input":"2021-06-10T19:18:13.005701Z","iopub.status.idle":"2021-06-10T19:18:13.015208Z","shell.execute_reply.started":"2021-06-10T19:18:13.005657Z","shell.execute_reply":"2021-06-10T19:18:13.014023Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#combined data cleaning function\ndef data_cleaning(question):\n    question = clean_tag(question)\n    question = clean_contractions(question)\n    question = clean_spec_chars(question)\n    question = lemmatize_text(question)\n    question = clean_latex_formula(question)\n    question = clean_numbers(question)\n    return question","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:13.017045Z","iopub.execute_input":"2021-06-10T19:18:13.017379Z","iopub.status.idle":"2021-06-10T19:18:13.028635Z","shell.execute_reply.started":"2021-06-10T19:18:13.017350Z","shell.execute_reply":"2021-06-10T19:18:13.027817Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Process question data on traing and validation dataset\nresampled_dataset['question_text'] = resampled_dataset['question_text'].progress_map(lambda resampled_dataset: data_cleaning(resampled_dataset))\nvalidation_data['question_text']=validation_data['question_text'].progress_map(lambda x: data_cleaning(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:18:13.029618Z","iopub.execute_input":"2021-06-10T19:18:13.029891Z","iopub.status.idle":"2021-06-10T19:26:18.314194Z","shell.execute_reply.started":"2021-06-10T19:18:13.029865Z","shell.execute_reply":"2021-06-10T19:26:18.313247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Spliting test & train data","metadata":{}},{"cell_type":"code","source":"x_train, x_test, y_train, y_test = train_test_split(\n    resampled_dataset['question_text'], resampled_dataset['target'], test_size=0.2, random_state=0)\nprint('x_train: ', x_train.shape, y_train.shape)\nprint('x_test: ',x_test.shape, y_test.shape)","metadata":{"scrolled":true,"_uuid":"3dbc763f4a123450ddb0c1c467be33f04625dfce","execution":{"iopub.status.busy":"2021-06-10T19:26:18.315495Z","iopub.execute_input":"2021-06-10T19:26:18.315796Z","iopub.status.idle":"2021-06-10T19:26:18.427061Z","shell.execute_reply.started":"2021-06-10T19:26:18.315767Z","shell.execute_reply":"2021-06-10T19:26:18.425995Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Create word counting vector train, test\n- Word counting vector: turn sentence in to words and number of ocurrance of that word in the sentence\n- Classify questions on the basis of grammar independency\n- Train on both vocabulary of train & test dataset, since the counting vectors might have to encrypt words from test dataset if it doesn't appear on the train dataset.","metadata":{}},{"cell_type":"code","source":"vectorizer = CountVectorizer()\n# Train on both train & validation dataset\nvectorizer.fit(list(resampled_dataset['question_text'].values)+ list(resampled_dataset['question_text'].values))\n# Create counting vetors for train, test, validation datasets based on the learned vocabulary\nx_tr = vectorizer.transform(x_train) \nx_te = vectorizer.transform(x_test)\nx_val = vectorizer.transform(validation_data['question_text'])\nprint(x_tr.shape)\nprint(x_te.shape)\nprint(x_val.shape)","metadata":{"_uuid":"bc1feb442a5d439db3b20f2f7eda3ab4cb620d27","execution":{"iopub.status.busy":"2021-06-10T19:26:18.428392Z","iopub.execute_input":"2021-06-10T19:26:18.428963Z","iopub.status.idle":"2021-06-10T19:32:25.796016Z","shell.execute_reply.started":"2021-06-10T19:26:18.428918Z","shell.execute_reply":"2021-06-10T19:32:25.795153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Logistic Regression\n- After obtaining counting vectors, put them through the model to train\n- Logistic Regression is used because of its simplicity and superior performance compare to other models such as: Random Forest Classifier, Naive Bayes, SVM","metadata":{}},{"cell_type":"code","source":"# import models\nfrom sklearn.metrics import accuracy_score, f1_score, classification_report\nfrom sklearn.linear_model import LogisticRegression","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:32:25.797044Z","iopub.execute_input":"2021-06-10T19:32:25.797495Z","iopub.status.idle":"2021-06-10T19:32:25.801358Z","shell.execute_reply.started":"2021-06-10T19:32:25.797463Z","shell.execute_reply":"2021-06-10T19:32:25.800473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Optimize parameters by GridSearchCV\n- Parameter C is part of model's input, GridSearchCV search for the best C value of the model","metadata":{}},{"cell_type":"code","source":"# Find C parameter\nfrom sklearn.model_selection import  GridSearchCV\nparams = {'C': [0.1, 1, 2, 3, 4, 5, 10]}\n\ngridsearch = GridSearchCV(LogisticRegression(), params, scoring='f1', n_jobs=-1, verbose=1)\ngridsearch.fit(x_tr, y_train)\nprint(gridsearch.best_params_)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:32:25.802391Z","iopub.execute_input":"2021-06-10T19:32:25.802785Z","iopub.status.idle":"2021-06-10T19:34:03.201698Z","shell.execute_reply.started":"2021-06-10T19:32:25.802757Z","shell.execute_reply":"2021-06-10T19:34:03.200352Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nmodel = LogisticRegression(C=gridsearch.best_params_['C'])\n## Run model\nprint(f\"Running Logistic Regression\")\nmodel.fit(x_tr, y_train)\n\ntrain_predictions = model.predict(x_tr)\ntrain_acc = accuracy_score(y_train, train_predictions)\ntrain_f1 = f1_score(y_train, train_predictions)\nprint(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}\") \ntest_predictions = model.predict(x_te)\ntest_acc = accuracy_score(y_test, test_predictions) \ntest_f1 = f1_score(y_test, test_predictions) \nprint(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:34:03.203781Z","iopub.execute_input":"2021-06-10T19:34:03.204329Z","iopub.status.idle":"2021-06-10T19:34:16.732676Z","shell.execute_reply.started":"2021-06-10T19:34:03.204253Z","shell.execute_reply":"2021-06-10T19:34:16.731738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Print out confusion matrix, classification_report\nimport seaborn as sns\nfrom sklearn.metrics import confusion_matrix\nsns.set(font_scale=1.4)\nsns.heatmap(pd.DataFrame(confusion_matrix(y_test, test_predictions), range(2),range(2)), annot=True, fmt='g')\nprint(classification_report(y_test, test_predictions))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:34:16.733961Z","iopub.execute_input":"2021-06-10T19:34:16.734228Z","iopub.status.idle":"2021-06-10T19:34:17.167754Z","shell.execute_reply.started":"2021-06-10T19:34:16.734202Z","shell.execute_reply":"2021-06-10T19:34:17.166605Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Observation:\n- F1 and accuracy is quite high\n- F1 and accuracy is close to each other once the dataset is balanced\n- Model's F1 when predicting insincere questions increase substantially when dataset is balanced\n- Model's accuracy when predicting sincere questions decrease (trivially) when dataset is balanced","metadata":{}},{"cell_type":"markdown","source":"## Submission\n- Trainging complete, run model on the validation dataset and save for submission","metadata":{}},{"cell_type":"code","source":"# Submission\nvalidation_predictions = model.predict(x_val)\nsubmission = pd.DataFrame({'qid':validation_data['qid'], 'prediction':validation_predictions })\nsubmission.to_csv('submission.csv', index=False)\nsubmission","metadata":{"execution":{"iopub.status.busy":"2021-06-10T19:34:17.169448Z","iopub.execute_input":"2021-06-10T19:34:17.169865Z","iopub.status.idle":"2021-06-10T19:34:18.119060Z","shell.execute_reply.started":"2021-06-10T19:34:17.169820Z","shell.execute_reply":"2021-06-10T19:34:18.117572Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### F1 score on validation dataset: ~0.6\n### F1 score on validation dataset is much lower than of test & train dataset but quite good in comparison to linear model","metadata":{}}]}