{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Quora Insincere Questions Classification: 2","metadata":{"id":"P6TzjzwJ5YuM"}},{"cell_type":"markdown","source":"### Loading Libraries","metadata":{"id":"IibKbRza5SeP"}},{"cell_type":"code","source":"!pip install regex eli5 emoji","metadata":{"id":"OICt4M8u5SeQ","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\n\nimport os\nimport re\nimport csv\nimport string\nimport emoji\nimport regex\nimport eli5\nimport pickle\nimport gensim\nimport spacy\nimport gc\nfrom tqdm import tqdm\nimport random\n\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom collections import Counter\nimport matplotlib.pyplot as plt\nfrom wordcloud import WordCloud, STOPWORDS\nfrom scipy.sparse import hstack\nfrom IPython.display import Image\nfrom prettytable import PrettyTable\n\nfrom tqdm import tqdm_notebook\ntqdm_notebook().pandas()\n\nfrom nltk.stem import PorterStemmer, SnowballStemmer, WordNetLemmatizer\nfrom nltk.stem.lancaster import LancasterStemmer\nfrom nltk.util import ngrams\n\nfrom sklearn.metrics import confusion_matrix, log_loss\nfrom sklearn.model_selection import train_test_split, KFold, GridSearchCV, RandomizedSearchCV\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.metrics import f1_score, classification_report\nfrom sklearn.calibration import CalibratedClassifierCV","metadata":{"id":"isqWlj_p5SeR","outputId":"a73b94d8-fa41-4144-8ee3-9fcdab9bcc43","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import nltk\nnltk.download('wordnet')\nnltk.download('punkt')","metadata":{"id":"SfAGU_nb5SeU","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Loading data","metadata":{"id":"-SVjgBTm5o9D"}},{"cell_type":"code","source":"path = '' #'../input/'\n\ndf_train = pd.read_csv(path+'train.csv')\ndf_test = pd.read_csv(path+'test.csv')\n\nprint(\"Number of data points in training data:\", df_train.shape[0])\nprint(\"Number of data points in test data:\", df_test.shape[0])","metadata":{"id":"z-wv-iqL4-2z","outputId":"0f4cbe4c-0570-4aae-da5f-f755f43797c9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head()","metadata":{"id":"LI_tKBRK4-24","outputId":"d9689e4f-fe97-42e9-b1a5-6e189a50e4d6","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preprocessing and cleaning","metadata":{"id":"wg9cB2qP60c_"}},{"cell_type":"markdown","source":"- Replacing math equations and url's with common abbrevation.\n- Cleaning contractions.\n- Spell Correction.\n- Removing punctuations.\n- Removing Stopwords.\n- Using WordNet Lemmatizer","metadata":{"id":"S-HsERsXQ1mU"}},{"cell_type":"code","source":"# Replacing math equations and url addresses with tags.\n# https://www.kaggle.com/canming/ensemble-mean-iii-64-36\ndef clean_tag(x):\n  if '[math]' in x:\n    x = re.sub('\\[math\\].*?math\\]', 'MATH EQUATION', x) #replacing with [MATH EQUATION]\n    \n  if 'http' in x or 'www' in x:\n    x = re.sub('(?:(?:https?|ftp):\\/\\/)?[\\w/\\-?=%.]+\\.[\\w/\\-?=%.]+', 'URL', x) #replacing with [url]\n  return x","metadata":{"id":"GH6BniD7ABhN","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clean_punct\n# https://www.kaggle.com/canming/ensemble-mean-iii-64-36\n\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', \n        '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', \n        '█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', \n        '—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞', \n        '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━', \n        '⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○', \n        '◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐', \n        '├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\\x97', '↺', \n        '∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊', \n        '┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´', \n        '↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\\x93', '≧', '］', \n        '\\x94', '∀', '♛', '\\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％', \n        '╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭', \n        '┊', '╯', '☾', '➔', '∴', '\\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐', \n        '▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\\x91', '➚', '◡', '╰', '\\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒', \n        '☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜', \n        '‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢', \n        '≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥', \n        '⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ', \n        '✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦', \n        '◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ', \n        '⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋', \n        '､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞', \n        '◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ', \n        '✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱', \n        '☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹', \n        '⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚', \n        '☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛', \n        '⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙', \n        'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘', \n        '˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛', \n        '☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑', \n        '✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ', \n        '⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘', \n        '⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊', \n        '⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷', \n        '◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝', \n        '⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢', \n        '≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']\n\ndef clean_punct(x):\n  x = str(x)\n  for punct in puncts:\n    if punct in x:\n      x = x.replace(punct, ' ')\n    return x","metadata":{"id":"yGinDy5763K-","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# correct_mispell\n# https://www.kaggle.com/oysiyl/107-place-solution-using-public-kernel\n\nmispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'bitcoin', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', \n                'electroneum':'bitcoin','nanodegree':'degree','hotstar':'star','dream11':'dream','ftre':'fire','tensorflow':'framework','unocoin':'bitcoin',\n                'lnmiit':'limit','unacademy':'academy','altcoin':'bitcoin','altcoins':'bitcoin','litecoin':'bitcoin','coinbase':'bitcoin','cryptocurency':'cryptocurrency',\n                'simpliv':'simple','quoras':'quora','schizoids':'psychopath','remainers':'remainder','twinflame':'soulmate','quorans':'quora','brexit':'demonetized',\n                'iiest':'institute','dceu':'comics','pessat':'exam','uceed':'college','bhakts':'devotee','boruto':'anime',\n                'cryptocoin':'bitcoin','blockchains':'blockchain','fiancee':'fiance','redmi':'smartphone','oneplus':'smartphone','qoura':'quora','deepmind':'framework','ryzen':'cpu','whattsapp':'whatsapp',\n                'undertale':'adventure','zenfone':'smartphone','cryptocurencies':'cryptocurrencies','koinex':'bitcoin','zebpay':'bitcoin','binance':'bitcoin','whtsapp':'whatsapp',\n                'reactjs':'framework','bittrex':'bitcoin','bitconnect':'bitcoin','bitfinex':'bitcoin','yourquote':'your quote','whyis':'why is','jiophone':'smartphone',\n                'dogecoin':'bitcoin','onecoin':'bitcoin','poloniex':'bitcoin','7700k':'cpu','angular2':'framework','segwit2x':'bitcoin','hashflare':'bitcoin','940mx':'gpu',\n                'openai':'framework','hashflare':'bitcoin','1050ti':'gpu','nearbuy':'near buy','freebitco':'bitcoin','antminer':'bitcoin','filecoin':'bitcoin','whatapp':'whatsapp',\n                'empowr':'empower','1080ti':'gpu','crytocurrency':'cryptocurrency','8700k':'cpu','whatsaap':'whatsapp','g4560':'cpu','payymoney':'pay money',\n                'fuckboys':'fuck boys','intenship':'internship','zcash':'bitcoin','demonatisation':'demonetization','narcicist':'narcissist','mastuburation':'masturbation',\n                'trignometric':'trigonometric','cryptocurreny':'cryptocurrency','howdid':'how did','crytocurrencies':'cryptocurrencies','phycopath':'psychopath',\n                'bytecoin':'bitcoin','possesiveness':'possessiveness','scollege':'college','humanties':'humanities','altacoin':'bitcoin','demonitised':'demonetized',\n                'brasília':'brazilia','accolite':'accolyte','econimics':'economics','varrier':'warrier','quroa':'quora','statergy':'strategy','langague':'language',\n                'splatoon':'game','7600k':'cpu','gate2018':'gate 2018','in2018':'in 2018','narcassist':'narcissist','jiocoin':'bitcoin','hnlu':'hulu','7300hq':'cpu',\n                'weatern':'western','interledger':'blockchain','deplation':'deflation', 'cryptocurrencies':'cryptocurrency', 'bitcoin':'blockchain cryptocurrency',}\n\ndef correct_mispell(x):\n  words = x.split()\n  for i in range(0, len(words)):\n    if mispell_dict.get(words[i]) is not None:\n      words[i] = mispell_dict.get(words[i])\n    elif mispell_dict.get(words[i].lower()) is not None:\n      words[i] = mispell_dict.get(words[i].lower())\n        \n  words = \" \".join(words)\n  return words","metadata":{"id":"aiEnsv5363Hn","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# remove stopwords\ndef remove_stopwords(x):\n  x = [word for word in x.split() if word not in STOPWORDS]\n  x = ' '.join(x)\n\n  return x","metadata":{"id":"gG0IuCDL8CXZ","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# clean word contractions\n## https://www.kaggle.com/theoviel/improve-your-score-with-text-preprocessing-v2 \n\ncontraction_mapping = {\"We'd\": \"We had\", \"That'd\": \"That had\", \"AREN'T\": \"Are not\", \"HADN'T\": \"Had not\", \"Could've\": \"Could have\", \"LeT's\": \"Let us\", \"How'll\": \"How will\", \"They'll\": \"They will\", \"DOESN'T\": \"Does not\", \"HE'S\": \"He has\", \"O'Clock\": \"Of the clock\", \"Who'll\": \"Who will\", \"What'S\": \"What is\", \"Ain't\": \"Am not\", \"WEREN'T\": \"Were not\", \"Y'all\": \"You all\", \"Y'ALL\": \"You all\", \"Here's\": \"Here is\", \"It'd\": \"It had\", \"Should've\": \"Should have\", \"I'M\": \"I am\", \"ISN'T\": \"Is not\", \"Would've\": \"Would have\", \"He'll\": \"He will\", \"DON'T\": \"Do not\", \"She'd\": \"She had\", \"WOULDN'T\": \"Would not\", \"She'll\": \"She will\", \"IT's\": \"It is\", \"There'd\": \"There had\", \"It'll\": \"It will\", \"You'll\": \"You will\", \"He'd\": \"He had\", \"What'll\": \"What will\", \"Ma'am\": \"Madam\", \"CAN'T\": \"Can not\", \"THAT'S\": \"That is\", \"You've\": \"You have\", \"She's\": \"She is\", \"Weren't\": \"Were not\", \"They've\": \"They have\", \"Couldn't\": \"Could not\", \"When's\": \"When is\", \"Haven't\": \"Have not\", \"We'll\": \"We will\", \"That's\": \"That is\", \"We're\": \"We are\", \"They're\": \"They' are\", \"You'd\": \"You would\", \"How'd\": \"How did\", \"What're\": \"What are\", \"Hasn't\": \"Has not\", \"Wasn't\": \"Was not\", \"Won't\": \"Will not\", \"There's\": \"There is\", \"Didn't\": \"Did not\", \"Doesn't\": \"Does not\", \"You're\": \"You are\", \"He's\": \"He is\", \"SO's\": \"So is\", \"We've\": \"We have\", \"Who's\": \"Who is\", \"Wouldn't\": \"Would not\", \"Why's\": \"Why is\", \"WHO's\": \"Who is\", \"Let's\": \"Let us\", \"How's\": \"How is\", \"Can't\": \"Can not\", \"Where's\": \"Where is\", \"They'd\": \"They had\", \"Don't\": \"Do not\", \"Shouldn't\":\"Should not\", \"Aren't\":\"Are not\", \"ain't\": \"is not\", \"What's\": \"What is\", \"It's\": \"It is\", \"Isn't\":\"Is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\n\ndef clean_contractions(text):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    \n    text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(\" \")])\n    return text","metadata":{"id":"yUvNn9jW8sMX","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# word lemmatizing\n\nlemmatizer = WordNetLemmatizer()\ndef lemma_text(x):\n  x = x.split()\n  x = [lemmatizer.lemmatize(word) for word in x]\n  x = ' '.join(x)\n\n  return x","metadata":{"id":"NtFry3bK88Z_","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_cleaning(x):\n  x = clean_tag(x)\n  x = clean_punct(x)\n  x = correct_mispell(x)\n  x = remove_stopwords(x)\n  x = clean_contractions(x)\n  x = lemma_text(x)\n  return x","metadata":{"id":"gXBCN3Oo9dcq","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# preprocessing given train and test data\ndf_train['preprocessed_question_text'] = df_train['question_text'].progress_map(lambda x: data_cleaning(x))\ndf_test['preprocessed_question_text'] = df_test['question_text'].progress_map(lambda x: data_cleaning(x))","metadata":{"id":"k8mloM359dSt","outputId":"32059bda-f63a-4ffc-838c-aa3fdca31695","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Basic Feature Extraction","metadata":{"id":"-o2KqO-cG2YD"}},{"cell_type":"code","source":"# Number of words\ndf_train['num_words'] = df_train['question_text'].apply(lambda x: len(str(x).split()))\ndf_test['num_words'] = df_test['question_text'].apply(lambda x: len(str(x).split()))\n\n# Number of capital_letters\ndf_train['num_capital_let'] = df_train['question_text'].apply(lambda x: len([c for c in str(x) if c.isupper()]))\ndf_test['num_capital_let'] = df_test['question_text'].apply(lambda x: len([c for c in str(x) if c.isupper()]))\n\n# Number of unique words\ndf_train['num_unique_words'] = df_train['question_text'].apply(lambda x: len(set(str(x).split())))\ndf_test['num_unique_words'] = df_test['question_text'].apply(lambda x: len(set(str(x).split())))\n\n# Number of characters\ndf_train['num_char'] = df_train['question_text'].apply(lambda x: len(str(x)))\ndf_test['num_char'] = df_test['question_text'].apply(lambda x: len(str(x)))\n\n# Number of stopwords\ndf_train['num_stopwords'] = df_train['question_text'].apply(lambda x: len([c for c in str(x).lower().split() if c in STOPWORDS]))\ndf_test['num_stopwords'] = df_test['question_text'].apply(lambda x: len([c for c in str(x).lower().split() if c in STOPWORDS]))\n\ndf_train.head()","metadata":{"id":"-NF51YEpHI41","outputId":"c4e6c4fc-609c-44a9-cf39-d787a40d04c9","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Train, Test & Val split","metadata":{"id":"59xa5YkyP9JP"}},{"cell_type":"code","source":"y = df_train['target'].values\nX_train, X_test, y_train, y_test = train_test_split(df_train, y, test_size=0.2, random_state=2019)\nX_train, X_val, y_train, y_val = train_test_split(X_train, y_train, test_size=0.1, random_state=2019)\n\nprint('X_train: ', X_train.shape, y_train.shape)\nprint('X_test: ',X_test.shape, y_test.shape)\nprint('X_val: ',X_val.shape, y_val.shape)","metadata":{"id":"xswiSfSKP9JT","outputId":"080d0c2b-7391-41ac-fe48-a23cdd850c5c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# check for percentage of class disb in train test split.\n\ndef plot_class_disb(class_disb, data, disb_name):\n  class_disb.plot(kind=\"bar\")\n  plt.xlabel('class')\n  plt.ylabel('Datapoints per class')\n  plt.title(f'Distribution of yi in {disb_name}')\n  plt.grid(True)\n  \n  sorted_yi = np.argsort(-class_disb.values)\n  print(disb_name,':')\n  for i in sorted_yi:\n    print('Number of data points in class', i, ':', class_disb.values[i], '(', np.round((class_disb.values[i]/data.shape[0]*100), 3), '%)')\n  \n  print('-'*50)","metadata":{"id":"WIkZQaViP9Jb","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# distribution of yi in train, test and val\n\nplt.subplot(1, 3, 1)\nplot_class_disb(\n  X_train['target'].value_counts().sort_values(),\n  X_train,\n  'TRAIN')\n         \nplt.subplot(1, 3, 2)\nplot_class_disb(\n  X_test['target'].value_counts().sort_values(),\n  X_test,\n  'TEST')\n\nplt.subplot(1, 3, 3)\nplot_class_disb(\n  X_val['target'].value_counts().sort_values(),\n  X_val,\n  'VAILIDATION')\nplt.subplots_adjust(right=2.0)\nplt.subplots_adjust(top=1)\nplt.show()","metadata":{"id":"R2lLb56eP9Jd","outputId":"fe951eb2-2a45-4e0a-cc77-0b48587314c3","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Baseline Model(LR) with basic extracted features","metadata":{"id":"RAO0b45ZQWZS"}},{"cell_type":"code","source":"tfidf = TfidfVectorizer(stop_words='english', ngram_range=(1, 3))\ntfidf.fit_transform(list(df_train['preprocessed_question_text'].values) + list(df_test['preprocessed_question_text'].values))\n\nX_train_ques = tfidf.transform(X_train['preprocessed_question_text'].values)\nX_test_ques = tfidf.transform(X_test['preprocessed_question_text'].values)\nX_val_ques = tfidf.transform(X_val['preprocessed_question_text'].values)\n\nprint(X_train_ques.shape)\nprint(X_test_ques.shape)\nprint(X_val_ques.shape)","metadata":{"id":"nNb8-zlPQWZd","outputId":"54cae894-0f46-44e8-a30f-72dc7e1dd8e8","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Standardize stats features\nfrom sklearn.preprocessing import StandardScaler\n\n# number of words\nnum_words =  StandardScaler()\nX_train_num_words = num_words.fit_transform(X_train['num_words'].values.reshape(-1, 1))\nX_test_num_words = num_words.transform(X_test['num_words'].values.reshape(-1, 1))\nX_val_num_words = num_words.transform(X_val['num_words'].values.reshape(-1, 1))\n\n# number of unique words\nnum_unique_words =  StandardScaler()\nX_train_num_unique_words = num_unique_words.fit_transform(X_train['num_unique_words'].values.reshape(-1, 1))\nX_test_num_unique_words = num_unique_words.transform(X_test['num_unique_words'].values.reshape(-1, 1))\nX_val_num_unique_words = num_unique_words.transform(X_val['num_unique_words'].values.reshape(-1, 1))\n\n# number of char\nnum_char =  StandardScaler()\nX_train_num_char = num_char.fit_transform(X_train['num_char'].values.reshape(-1, 1))\nX_test_num_char = num_char.transform(X_test['num_char'].values.reshape(-1, 1))\nX_val_num_char = num_char.transform(X_val['num_char'].values.reshape(-1, 1))\n\n# number of stopwords\nnum_stopwords =  StandardScaler()\nX_train_num_stopwords = num_stopwords.fit_transform(X_train['num_stopwords'].values.reshape(-1, 1))\nX_test_num_stopwords = num_stopwords.transform(X_test['num_stopwords'].values.reshape(-1, 1))\nX_val_num_stopwords = num_stopwords.transform(X_val['num_stopwords'].values.reshape(-1, 1))","metadata":{"id":"jti8vx_fQWZo","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Stacking features \n\nX_tr = hstack((\n    X_train_ques,\n    X_train_num_words,\n    X_train_num_unique_words,\n    X_train_num_char,\n    X_train_num_stopwords\n))\n\nX_te = hstack((\n    X_test_ques,\n    X_test_num_words,\n    X_test_num_unique_words,\n    X_test_num_char,\n    X_test_num_stopwords\n))\n\nX_cv = hstack((\n    X_val_ques,\n    X_val_num_words,\n    X_val_num_unique_words,\n    X_val_num_char,\n    X_val_num_stopwords\n))\n\nprint(X_tr.shape, y_train.shape)\nprint(X_te.shape, y_test.shape)\nprint(X_cv.shape, y_val.shape)","metadata":{"id":"ote82PP0QWZq","outputId":"956c0522-0bc3-4c74-fb6b-08d1e9edb213","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"parmas = {'C': [0.001, 0.001, 0.1, 1, 10]}\n\ngridsearch = GridSearchCV(LogisticRegression(), parmas, scoring='f1', n_jobs=-1, verbose=1)\ngridsearch.fit(X_tr, y_train)","metadata":{"id":"KKKXGsGoQWZv","outputId":"c80b6ca3-d823-4145-e4e0-51b690981ebb","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"gridsearch.best_params_","metadata":{"id":"AfoSIoeYY87g","outputId":"5b7c4462-b972-4f15-d0b1-7a8bffa6c274","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"clf = LogisticRegression(C=1)\nclf.fit(X_tr, y_train)","metadata":{"id":"YdlZgQeiQWZ2","outputId":"31008baf-0b3d-4525-e849-d65e184ff3e7","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pred_val = clf.predict_proba(X_cv)[:,1]\n\nfor t in np.arange(0.1, 0.201, 0.01):\n  t = np.round(t, 2)\n  print('F1 score at threshold {0} is {1}'.format(t, f1_score(y_val, (y_pred_val>t).astype('int'))))","metadata":{"id":"SuGD2Q1NQWZ6","outputId":"146ba8ec-3f17-4944-e37b-5713ca238c1e","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"t = 0.17\ny_pred_test = clf.predict_proba(X_te)[:,1]\nprint('F1-Score: ', round(f1_score(y_test, (y_pred_test>t)), 5))","metadata":{"id":"dkc9_TyFQWZ9","outputId":"a03fcca7-c7bf-4b95-e57e-a337c45f4230","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def plot_confusion_matrix(y_test, y_pred):\n  cm = confusion_matrix(y_test, y_pred)\n  plt.figure(figsize=(5, 3))\n  sns.heatmap(cm, annot=True, fmt='d')\n  plt.title('Confusion Matrix')\n  plt.show()\n  \n  print(f\"Correctly classified sincere questions: {round(cm[0][0]/(cm[0][0] + cm[0][1])*100, 2)}%\")\n  print(f\"Correctly classified insincere questions: {round(cm[1][1]/(cm[1][0] + cm[1][1])*100, 2)}%\")","metadata":{"id":"FCjF4X3lQWaG","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"plot_confusion_matrix(y_test, (y_pred_test>t))","metadata":{"id":"oKfvuda2QWaJ","outputId":"59a28ec6-bd93-44d4-b4f9-f705c6e5edb4","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(classification_report(y_test, (y_pred_test>t)))","metadata":{"id":"T_0lFR2fQWaN","outputId":"f40d1f46-cee4-4672-ec67-3f2fe0a38a4c","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"eli5.show_weights(clf, vec=tfidf, top=50, feature_filter=lambda x:x != '<BIAS>')","metadata":{"id":"mQSRdpZNQWaR","outputId":"835b2b6a-1c5b-4830-e92f-894d00353b26","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def display_incorrect_predictions(data, pred):\n  \"\"\"\n  Displays text for both correct|incorrect pred for classes(0, 1)\n  \"\"\"\n  \n  df_classified = data.copy()\n  df_classified['pred'] = (pred>t)\n  \n  print('\\033[1m'+'Sample Class Prediction Anaylsis')\n  print('')\n  \n  print('\\033[1m'+'Sincere question:\\033[34m Correct prediction')\n  print('\\033[0m')\n  print(data[(df_classified['pred'] == False) & (df_classified['target'] ==0) & (df_classified['num_words'] <15)]['question_text'].head(5).values)\n  print('\\033[1m'+'Sincere question:\\033[31m Incorrect prediction')\n  print('\\033[0m')\n  print(data[(df_classified['pred'] == True) & (df_classified['target'] ==0) & (df_classified['num_words'] <15)]['question_text'].head(5).values)\n  \n  print('-'*50)\n  \n  print('\\033[1m'+'Insincere question :\\033[34m Correct prediction')\n  print('\\033[0m')\n  print(data[(df_classified['pred'] == True) & (df_classified['target'] ==1) & (df_classified['num_words'] <15)]['question_text'].head(5).values)\n  print('\\033[1m'+'Insincere question:\\033[31m Incorrect Prediction')\n  print('\\033[0m')\n  print(data[(df_classified['pred'] == False) & (df_classified['target'] ==1) & (df_classified['num_words'] <15)]['question_text'].head(5).values)\n  ","metadata":{"id":"wyD4I-sBQWaW","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"display_incorrect_predictions(X_test, (y_pred_test>t))","metadata":{"id":"HGub20zFQWaX","outputId":"a4d201e8-48b8-4224-bbb8-1ada0aba2633","trusted":true},"execution_count":null,"outputs":[]}]}