{"cells":[{"metadata":{},"cell_type":"markdown","source":"Quora Insincere Question Classification
"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Importing Libraries
"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import re\nimport os\nimport nltk\nimport time\nimport scipy\nimport string\nimport operator\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.pyplot as pt\nimport plotly.graph_objects as go\nfrom collections import defaultdict\nfrom wordcloud import WordCloud, STOPWORDS\nfrom plotly.offline import download_plotlyjs, init_notebook_mode, plot, iplot\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer, WordNetLemmatizer\nfrom nltk.tokenize import sent_tokenize, word_tokenize\nimport warnings\nwarnings.filterwarnings('ignore')\nfrom sklearn import metrics\nfrom mlxtend.plotting import plot_confusion_matrix\nfrom sklearn.model_selection import RandomizedSearchCV, GridSearchCV, RandomizedSearchCV, train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading Dataset
"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Loading the train and test data\"\"\"\nquora_train = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\nquora_test = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Text Preprocessing
"},{"metadata":{"trusted":true},"cell_type":"code","source":"punctuation = \"=∈•→≈≠(∗☹／¼؟・̑℃צ\\\\⊂⎠…∫̴̀）☁|´{∂′˜¥⅔▾∠®ு¢⅓✔≡↑│΅̶̊/∅¬–☉∨（̸̪＄،！，∞˂≥&)✏∩̉-↓”?≅̂÷″⟨̃+̵✓♡̳;̈\\\n[ா¾*͂:＞⋅°]_£♨✌☝#।♏€＝±⎛℅∴⟩ி—̎≤ْ̣̅⎝‘«@̌⋯△̱∪†、̲%̕◦¿－∘$̡̓♣⎞^∆<⊆̗.❤；’∼„❓·™,̖»½⁻♀¸‛©¡̷̐‰∑√⇒>：˚✅்̾`⊥∇͡!̄➡~̿\\x92¶\\\n☺§−♭。▒“？∝⬇＾¯◌}\\'∀∧\\x02্ു̧̤͆ีॄ\\xad̋ः̘《͜\\u200cি̫\\u2060ุ͗ూ⦁͑\\u202a\\x7fូ้ାู̰̼̺͒₦\\x06ٌ͋ះం͚ె្⌚్₱㏑া́ा़̩͌₩\\uf0d8〖ͅ⊨ֿ〗ਾ̍️「ॣ\\x8d่ी㏒͐োோ̦〇」̥\\\n\\u200bં\\x13∛͊\\x9d⚧》͈̝̜ে̓∖ీా̭̔\\x8f͖ৃ\\u200eَ̒̈́͛ੰ\\ufeffા₹\\u200fॉைី਼͕̯͎︡ា\\ue019\\x1b͉ँ\\x10\\x01̮\\u2061्ೋ\\u202c\\\n‑ിे્͝ិ͘ै\\x17ੀ\\uf02d\\x1ă∡ীಿੁ⧽ៃ̽\\x03̛̙ंి᠌ਿ≱⧼ั್ِ͠ू̀̚₊\\ue01bಾौើំ়͇̞͔⃗ोृ̬ొು്̻︠ు̟ਂ̢ാॢिௌ̹ुّ‐\"\n\ncontractions = {\"'aight\": 'alright', \"ain't\": 'am not', \"amn't\": 'am not', \"aren't\": 'are not', \n \"can't\": 'can not', \"'cause\": 'because', \"could've\": 'could have', \"couldn't\": 'could not', \n \"couldn't've\": 'could not have', \"daren't\": 'dare not', \"daresn't\": 'dare not', \n \"dasn't\": 'dare not', \"didn't\": 'did not', \"doesn't\": 'does not', \"don't\": 'do not', \n 'dunno': \"don't know\", \"d'ye\": 'do you', \"e'er\": 'ever', \"everybody's\": 'everybody is', \n \"everyone's\": 'everyone is', 'finna': 'fixing to', \"g'day\": 'good day', 'gimme': 'give me', \n \"giv'n\": 'given', 'gonna': 'going to', \"gon't\": 'go not', 'gotta': 'got to', \n \"hadn't\": 'had not', \"had've\": 'had have', \"hasn't\": 'has not', \"haven't\": 'have not', \n \"he'd\": 'he had', \"he'll\": 'he will', \"he's\": 'he is', \"he've\": 'he have', \"how'd\": 'how did',\n 'howdy': 'how do you do', \"how'll\": 'how will', \"how're\": 'how are', \"how's\": 'how is', \n \"I'd\": 'I had', \"I'd've\": 'I would have', \"I'll\": 'I will', \"I'm\": 'I am', \n \"I'm'a\": 'I am about to', \"I'm'o\": 'I am going to', 'innit': 'is it not', \"I've\": 'I have', \n \"isn't\": 'is not', \"it'd \": 'it would', \"it'll\": 'it will', \"it's \": 'it is', \n 'iunno': \"I don't know\", \"let's\": 'let us', \"ma'am\": 'madam', \"mayn't\": 'may not', \n \"may've\": 'may have', 'methinks': 'me thinks', \"mightn't\": 'might not', \n \"might've\": 'might have', \"mustn't\": 'must not', \"mustn't've\": 'must not have', \n \"must've\": 'must have', \"needn't\": 'need not', 'nal': 'and all', \"ne'er\": 'never', \n \"o'clock\": 'of the clock', \"o'er\": 'over',\"ol'\": 'old', \"oughtn't\": 'ought not', \"'s\": 'is',\n \"shalln't\": 'shall not', \"shan't\": 'shall not', \"she'd\": 'she would', \"she'll\": 'she will', \n \"she's\": 'she is', \"should've\": 'should have', \"shouldn't\": 'should not', \"shouldn't've\": 'should not have', \"somebody's\": 'somebody has', \n \"someone's\": 'someone has', \"something's\": 'something has', \"so're\": 'so are', \"that'll\": 'that will', \n \"that're\": 'that are', \"that's\": 'that is', \"that'd\": 'that would', \"there'd\": 'there would', \n \"there'll\": 'there will', \"there're\": 'there are', \"there's\": 'there is', \"these're\": 'these are', \n \"they've\": 'they have', \"this's\": 'this is', \"those're\": 'those are', \"those've\": 'those have', \"'tis\": 'it is', \n \"to've\": 'to have', \"'twas\": 'it was', 'wanna': 'want to', \"wasn't\": 'was not', \"we'd\": 'we would', \n \"we'd've\": 'we would have', \"we'll\": 'we will', \"we're\": 'we are', \"we've\": 'we have', \"weren't\": 'were not', \n \"what'd\": 'what did', \"what'll\": 'what will', \"what're\": 'what are', \"what's\": 'what does', \"what've\": 'what have',\n \"when's\": 'when is', \"where'd\": 'where did', \"where'll\": 'where will', \"where're\": 'where are',\n \"where's\": 'where is',\"where've\": 'where have', \"which'd\": 'which would', \"which'll\": 'which will', \n \"which're\": 'which are',\"which's\": 'which is', \"which've\": 'which have', \"who'd\": 'who would',\n \"who'd've\": 'who would have', \"who'll\": 'who will', \"who're\": 'who are', \"who'ves\": 'who is', \"who'\": 'who have',\n \"why'd\": 'why did', \"why're\": 'why are', \"why's\": 'why does', \"willn't\": 'will not', \"won't\": 'will not',\n 'wonnot': 'will not', \"would've\": 'would have', \"wouldn't\": 'would not', \"wouldn't've\": 'would not have',\n \"y'all\": 'you all', \"y'all'd've\": 'you all would have', \"y'all'd'n've\": 'you all would not have',\n \"y'all're\": 'you all are', \"cause\":\"because\",\"have't\":\"have not\",\"cann't\":\"can not\",\"ain't\":\"am not\",\n \"you'd\": 'you would', \"you'll\": 'you will', \"you're\": 'you are', \"you've\": 'you have', 'cannot': 'can not', \n 'wont': 'will not', \"You'\": 'Am not', \"Ain'\": 'Am not', \"Amn'\": 'Am not', \"Aren'\": 'Are not',\n \"Can'\": 'Because', \"Could'\": 'Could have', \"Couldn'\": 'Could not have', \"Daren'\": 'Dare not', \n \"Daresn'\": 'Dare not', \"Dasn'\": 'Dare not', \"Didn'\": 'Did not', \"Doesn'\": 'Does not', \"Don'\": \"Don't know\", \n \"D'\": 'Do you', \"E'\": 'Ever', \"Everybody'\": 'Everybody is', \"Everyone'\": 'Fixing to', \"G'\": 'Give me', \n \"Giv'\": 'Going to', \"Gon'\": 'Got to', \"Hadn'\": 'Had not', \"Had'\": 'Had have', \"Hasn'\": 'Has not', \n \"Haven'\": 'Have not', \"He'\": 'He have', \"How'\": 'How is', \"I'\": 'I have', \"Isn'\": 'Is not', \"It'\": \"I don't know\", \n \"Let'\": 'Let us', \"Ma'\": 'Madam', \"Mayn'\": 'May not', \"May'\": 'Me thinks', \"Mightn'\": 'Might not', \n \"Might'\": 'Might have', \"Mustn'\": 'Must not have', \"Must'\": 'Must have', \"Needn'\": 'And all', \"Ne'\": 'Never',\n \"O'\": 'Old', \"Oughtn'\": 'Is', \"Shalln'\": 'Shall not', \"Shan'\": 'Shall not', \"She'\": 'She is', \n \"Should'\": 'Should have', \"Shouldn'\": 'Should not have', \"Somebody'\": 'Somebody has', \"Someone'\": 'Someone has', \n \"Something'\": 'Something has', \"So'\": 'So are', \"That'\": 'That would', \"There'\": 'There is',\n \"They'\": 'They have', \"This'\": 'This is', \"Those'\": 'It is', \"To'\": 'Want to', \"Wasn'\": 'Was not',\n \"Weren'\": 'Were not', \"What'\": 'What have', \"When'\": 'When is', \"Where'\": 'Where have', \"Which'\": 'Which have', \n \"Who'\": 'Who have', \"Why'\": 'Why does', \"Willn'\": 'Will not', \"Won'\": 'Will not', \"Would'\": 'Would have',\n \"Wouldn'\": 'Would not have', \"Y'\": 'You all are',\"What's\":\"What is\",\"What're\":\"What are\",\"what's\":\"what is\",\n \"what're\":\"what are\", \"Who're\":\"Who are\", \"your're\":\"you are\",\"you're\":\"you are\", \"You're\":\"You are\",\n \"We're\":\"We are\", \"These'\": 'These have', \"we're\":\"we are\",\"Why're\":\"Why are\",\"How're\":\"How are \",\n \"how're \":\"how are \",\"they're \":\"they are \", \"befo're\":\"before\",\"'re \":\" are \",'don\"t ':\"do not\", \n \"Won't \":\"Will not \",\"could't\":\"could not\", \"would't\":\"would not\", \"We'\": 'We have',\"Hasn't\":\"Has not\",\n \"n't\":\"not\", 'who\"s':\"who is\"}\n\ncorrect_words = dict({\"√\":\" sqrt \",\"π\":\" pi \",\"α\":\" alpha \",\"θ\":\" theta \",\"∞\":\" infinity \",\"∝\":\" proportional to \",\"sinx\":\" sin x \",\n \"cosx\":\" cos x \", \"tanx\":\" tan x \",\"cotx\":\" cot x \", \"secx\":\" sec x \", \"cosecx\":\" cosec x \", \"£\":\" pound \", \"β\":\" beta \", \n \"σ\": \" theta \", \"∆\":\" delta \",\"μ\":\" mu \",'∫': \" integration \", \"ρ\":\" rho \", \"λ\":\" lambda \",\"∩\":\" intersection \",\n \"Δ\":\" delta \", \"φ\":\" phi \", \"℃\":\" centigrade \",\"≠\":\" does not equal to \",\"Ω\":\" omega \",\"∑\":\" summation \",\"∪\":\" union \",\n \"ψ\":\" psi \", \"Γ\":\" gamma \",\"⇒\":\" implies \",\"∈\":\" is an element of \", \"≡\":\" is congruent to \",\"xⁿ\":\" x power n\",\n \"≈\":\" is approximately equal to \", \"~\":\" is distributed as \",\"≅\":\" is isomorphic to \",\"⩽\":\" is less than or equal to \",\n \"≥\":\" is greater than or equal to \",\"⇐\":\" is implied by \",\"⇔\":\" is equivalent to \", \"∉\":\" is not an element of \",\n \"∅\" : \" empty set \", \"∛\":\" cbrt \",\"÷\":\" division \",\"㏒\":\" log \",\"∇\":\" del \",\"⊆\":\" is a subset of \",\"±\":\" plus–minus \",\n \"⊂\":\" is a proper subset of \",\"€\":\" euro \",\"㏑\":\" ln \",\"₹\":\" rupee \",\"∀\":\" there exists \",\"∆\":\" delta \",\"∑\":\" summation\",\n \"=\":\" equal to \",\"₹\":\" rupee \",\"≤\":\" less than or equal to \", \"±\":\" plus or minus \", \"£\":\" pound \",\"∝\":\" propertional to \",\n \"¼\":\" one by four \",\"&\":\" and \",\"™\":\" trade mark \",\"½\":\" one by two \",\"＄\":\" dollar \",\"quorans\":\"quora\",\n \"cryptocurrencies\":\"cryptocurrency\",\"haveheard\":\"have heard\",\"amafraid\":\"am afraid\",\"amplanning\":\"am planning\",\n \"demonetisation\":\"demonetization\",\"pokémon\":\"pokemon\",\"havegot\":\"have got\",\"amscared\":\"am scared\",\"qoura\":\"quora\",\n \"haveread\":\"have heard\",\"fiancé\":\"fiance\", \"amworried\":\"am worried\",\"amfeeling\":\"am feeling\",\"havetried\":\"have tried\", \n \"amwriting\":\"am writing\",\"havealways\":\"have always\",\"amconfused\":\"am confused\", \"havejust\":\"have just\",\"amgay\":\"am gay\",\n \"amstudying\":\"am studying\",\"amtalking\":\"am talking\",\"amdepressed\":\"am depressed\",\"havenoticed\":\"have noticed\",\n \"amdating\":\"am dating\", \"x²\":\"x square\",\"quoras\":\"quora\",\"amcurious\":\"am curious\",\"havelost\":\"have lost\",\n \"amunable\":\"am unable\", \"haverecently\":\"have recently\", \"amasking\":\"am asking\", \"amsick\":\"am sick\", \"clickbait\":\"click bait\", \n \"haveever\": \"have ever\", \"amapplying\":\"am applying\", \"haveknown\":\"have known\",\"ampregnant\":\"am pregnant\",\n \"haveonly\":\"have only\",\"amalone\":\"am alone\",\"havestarted\":\"have started\", \"²\":\"square\", \"amlearning\":\"am learning\",\n \"amconstantly\":\"am constantly\", \"amugly\":\"am ugly\", \"amstruggling\":\"am struggling\", \"amready\":\"am ready\", \"são\":\"sao\", \n \"amturning\":\"am turning\", \"genderfluid\":\"gender fluid\", \"wouldrather\":\"would rather\", \"chapterwise\":\"chapter wise\", \n \"undergraduation\":\"under graduation\", \"blockchains\":\"blockchain\", \"amwondering\":\"am wondering\",\"havecompleted\":\"have completed\", \n \"amextremely\":\"am extremely\", \"amattracted\":\"am attracted\", \"amlosing\":\"am losing\", \"fiancée\":\"fiance\",\n \"amangry\":\"am angry\", \"amaddicted\":\"am addicted\", \"havegotten\":\"have gotten\", \"makaut\":\"make out\", \"havegotten\":\"have gotten\", \n \"amyoung\":\"am young\", \"amfalling\":\"am falling\", \"clichés\":\"cliches\", \"beyoncé\":\"beyonce\",\n \"erdoğan\":\"erdogan\", \"atatürk\":\"ataturk\", \"amfinding\":\"am finding\", \"ampreparing\":\"am preparing\", \"whyis\":\"why is\", \n \"haveused\":\"have used\", \"ammarried\":\"am married\", \"2k17\":\"2017\", \"cos2x\":\"cos 2x\", \"flipcart\":\"flipkart\", \n \"brexit\":\"britain exit\", \"havefallen\":\"have fallen\",\"demonitisation\":\"demonetization\", \"microservices\":\"micro services\",\n \"amallergic\":\"am allergic\", \"amskinny\":\"am skinny\", \"amaware\":\"am aware\",\"amdoing\":\"am doing\",\"amtired\":\"am tired\",\n \"p0rnographic\":\"pornographic\",\"1st\":\"first\",\"2nd\":\"second\",\"3rd\":\"third\",\"ww2\":\"www\",\"ps4\":\"play station four\"})\n\ndefined_stopwords = ['i','me', 'my', 'myself', 'we', 'our', 'ours', 'ourselves', 'you', \"you're\", \"you've\", \n \"you'll\", \"you'd\", 'your', 'yours', 'yourself', 'yourselves', 'he', 'him', 'his', \n 'himself', 'she', \"she's\", 'her', 'hers','herself', 'it', \"it's\", 'its', 'itself', \n 'they', 'them', 'their', 'theirs', 'themselves', 'what', 'which', 'or', 'because', 'as',\n 'until', 'while', 'of', 'at', 'by', 'for', 'with', 'about', 'against', 'between', 'into', \n 'who', 'whom', 'this', 'why','that', \"that'll\", 'these', 'those', 'am', 'is', 'are', \n 'was', 'were', 'be', 'been', 'the', 'and', 'but', 'if', 'through', 'during', 'before', \n 'after', 'being', 'have', 'has', 'had', 'having', 'do', 'does', 'did', 'doing', 'a', \n 'an', 'above', 'below', 'to', 'from', 'up', 'down', 'in', 'out', 'on', 'off',\n 'over', 'under', 'again', 'further', 'then', 'once', 'here', 'there', 'when', 'where',\n 'why', 'how', 'all', 'any', 'both', 'each', 'few', 'more', 'most', 'other', 'some', \n 'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too', 'very', 's',\n 't', 'u', 'can', 'will', 'just', 'don', \"don't\", 'should', \"should've\", 'now', 'd', \n 'll', 'm', 'o', 're', 've', 'y', 'ain', 'aren', \"aren't\", 'couldn', \"couldn't\", 'didn',\n \"didn't\", 'doesn', \"doesn't\", 'hadn', \"hadn't\", 'hasn', \"hasn't\", 'haven', \"haven't\", \n 'isn', \"isn't\", 'ma', 'mightn', \"mightn't\", 'mustn', \"mustn't\", 'needn', \"needn't\", \n 'shan', \"shan't\", 'shouldn', \"shouldn't\", 'wasn', \"wasn't\", 'weren', \"weren't\", 'won', \n \"won't\", 'wouldn', \"wouldn't\",\"would\", \"could\", 'the']\n\n\"\"\"mapping all questions text into string and making it lowercase\"\"\"\ndef preprocessing(text,stopwords,contractions,correct_words,punctuation):\n\n start_time = time.time()\n #removing punctuation\n translate_table = dict((ord(char), None) for char in punctuation) \n\n #convert all the words to lower case first and then remove the stopwords\n text = text.map(str)\n for line in range(len(text.values)):\n text.values[line] = text.values[line].lower()\n \n #decontraction\n for idx,val in enumerate(text.values):\n val = ' '.join(word.replace(word,contractions[word]) if word in contractions else word for \n word in val.split())\n #generic one\n val = re.sub(r\"\\'s\", \" \", val); val = re.sub(r\"\\''s\", \" \", val); val = re.sub(r\"\\\"s\", \" \", val);\n val = re.sub(r\"n\\''t\", \" not \", val); val = re.sub(r\"n\\\"t\", \" not \", val); \n val = re.sub(r\"\\'re \", \" are \", val); val = re.sub(r\"\\'d \", \" would\", val); \n val = re.sub(r\"\\''d \", \" would\", val); val = re.sub(r\"\\\"d \", \" would\", val);\n val = re.sub(r\"\\'ll \", \" will\", val); val = re.sub(r\"\\''ll \", \" will\", val); \n val = re.sub(r\"\\\"ll \", \" will\", val);val = re.sub(r\"\\'ve \", \" have\", val); \n val = re.sub(r\"\\''ve \", \" have\", val); val = re.sub(r\"\\\"ve \", \" have\", val);\n val = re.sub(r\"\\'m \", \" am\", val); val = re.sub(r\"\\''m \",\" am\", val); \n val = re.sub(r\"\\\"m \",\" am\", val); val = re.sub(\"''\",\"\",val); val = re.sub(\"``\",\"\",val);\n val = re.sub('\"','',val); val = re.sub(\"̇\",'',val); val = re.sub(\"\\s{2}\",\" \",val)\n \n #replacing correct word with incorrect one\n val = ' '.join(word.replace(word,correct_words[word]) if word in correct_words else word \n for word in val.split())\n \n #removing stopwords\n val = ' '.join(e.lower() for e in val.split() if e.lower() not in stopwords)\n \n #Removing special characters\n val = val.translate(translate_table)\n \n #mapping text after above steps\n text.values[idx] = val.strip() \n \n #lemmatization\n lemmatizer = WordNetLemmatizer()\n for idx,val in enumerate(text.values):\n sent = ''\n for word in word_tokenize(val):\n sent += lemmatizer.lemmatize(word) + ' '\n text.values[idx] = sent.strip()\n \n hours, rem = divmod(time.time()-start_time, 3600)\n print(\"Time to process the whole function : {:02}.{:0>2}.{:02} minutes.\".format(int(hours),\n int(divmod(rem, 60)[0]), \n int(divmod(rem, 60)[1])))\n\n return text ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"quora_train['question_text_preprocessed'] = preprocessing(quora_train['question_text'],defined_stopwords,\n contractions,correct_words,punctuation)\nquora_test['question_text_preprocessed'] = preprocessing(quora_test['question_text'],defined_stopwords,\n contractions,correct_words,punctuation)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Feature Engineering
"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"Here I am omitting diversity_score, total_punctuations and total_digit as these features \n are not kind of indentical for both class.\"\"\"\n\nstart_time = time.time()\n\n\"\"\"number of words in the question text\"\"\"\nquora_train[\"total_words\"] = quora_train[\"question_text\"].apply(lambda sent: len(str(sent).split()))\nquora_test[\"total_words\"] = quora_test[\"question_text\"].apply(lambda sent: len(str(sent).split()))\n\n\"\"\"number of characters in the question text\"\"\"\nquora_train[\"total_chars\"] = quora_train[\"question_text\"].apply(lambda sent: len(str(sent)))\nquora_test[\"total_chars\"] = quora_test[\"question_text\"].apply(lambda sent: len(str(sent)))\n\n\"\"\"total unique words in the question text\"\"\"\nquora_train[\"total_unique_words\"] = quora_train[\"question_text\"].apply(lambda sent: len(set(str(sent)\n .split())))\nquora_test[\"total_unique_words\"] = quora_test[\"question_text\"].apply(lambda sent: len(set(str(sent)\n .split())))\n\n\"\"\"word score in the question text\"\"\"\nquora_train['word_score'] = quora_train[\"total_unique_words\"]/quora_train[\"total_words\"]\nquora_test['word_score'] = quora_test[\"total_unique_words\"]/quora_test[\"total_words\"]\n\n\"\"\"total number of stopwords in the question text\"\"\"\n#gettings stopwords from nltk library\nStopwords = stopwords.words('english')\nquora_train[\"total_stopwords\"] = quora_train[\"question_text\"].apply(lambda sent: len([s for s in str(sent)\n .lower().split() if s \n in Stopwords]))\nquora_test[\"total_stopwords\"] = quora_test[\"question_text\"].apply(lambda sent: len([s for s in str(sent)\n .lower().split() if s \n in Stopwords]))\n\n\"\"\"total number of UPPERcase words in the question text\"\"\"\nquora_train[\"total_upper\"] = quora_train[\"question_text\"].apply(lambda sent: len([u for u in str(sent)\n .split() if u.isupper()]))\nquora_test[\"total_upper\"] = quora_test[\"question_text\"].apply(lambda sent: len([u for u in str(sent)\n .split() if u.isupper()]))\n\n\"\"\"total number of lowercase words in the question text\"\"\"\nquora_train[\"total_lower\"] = quora_train[\"question_text\"].apply(lambda sent: len([l for l in str(sent)\n .split() if l.islower()]))\nquora_test[\"total_lower\"] = quora_test[\"question_text\"].apply(lambda sent: len([l for l in str(sent)\n .split() if l.islower()]))\n\n\"\"\"total number of word title in the question text\"\"\"\nquora_train[\"total_word_title\"] = quora_train[\"question_text\"].apply(lambda sent: len([u for u in \n str(sent).split() \n if u.istitle()]))\nquora_test[\"total_word_title\"] = quora_test[\"question_text\"].apply(lambda sent: len([u for u in \n str(sent).split() \n if u.istitle()]))\n\n\n\"\"\"median word length of the question text\"\"\"\nquora_train[\"median_word_len\"] = quora_train[\"question_text\"].apply(lambda sent: np.median([len(w) \n for w in \n str(sent)\n .split()]))\nquora_test[\"median_word_len\"] = quora_test[\"question_text\"].apply(lambda sent: np.median([len(w) \n for w in \n str(sent)\n .split()]))\n\n\"\"\"Truncating Outliers\"\"\"\n#Total number of words\nquora_train['total_words'].loc[quora_train[\"total_words\"] > 60] = 60\nquora_test['total_words'].loc[quora_test[\"total_words\"] > 60] = 60\n#Total number of characters\nquora_train['total_chars'].loc[quora_train[\"total_chars\"] > 250] = 250\nquora_test['total_chars'].loc[quora_test[\"total_chars\"] > 250] = 250\n#Total number of unique words\nquora_train['total_unique_words'].loc[quora_train[\"total_unique_words\"] > 60] = 60\nquora_test['total_unique_words'].loc[quora_test[\"total_unique_words\"] > 60] = 60\n#Word Score\nquora_train['word_score'].loc[quora_train[\"word_score\"] < 0.6] = 0.6\nquora_test['word_score'].loc[quora_test[\"word_score\"] < 0.6] = 0.6\n#Total number of stopwords\nquora_train['total_stopwords'].loc[quora_train[\"total_stopwords\"] > 30] = 30\nquora_test['total_stopwords'].loc[quora_test[\"total_stopwords\"] > 30] = 30\n#Total number of uppercase word\nquora_train['total_upper'].loc[quora_train[\"total_upper\"] > 6] = 6\nquora_test['total_upper'].loc[quora_test[\"total_upper\"] > 6] = 6\n#Total number of lowercase word\nquora_train['total_lower'].loc[quora_train[\"total_lower\"] > 45] = 45\nquora_test['total_lower'].loc[quora_test[\"total_lower\"] > 45] = 45\n#Total number of word title\nquora_train['total_word_title'].loc[quora_train[\"total_word_title\"] > 15] = 15\nquora_test['total_word_title'].loc[quora_test[\"total_word_title\"] > 15] = 15\n#median_word_length\nquora_train['median_word_len'].loc[quora_train[\"median_word_len\"] > 10] = 10\nquora_test['median_word_len'].loc[quora_test[\"median_word_len\"] > 10] = 10\n\n#time calculation\nhours, rem = divmod(time.time()-start_time, 3600)\nprint(\"Time to process the whole function : {:02}.{:0>2}.{:02} minutes.\".format(int(hours),\n int(divmod(rem, 60)[0]), \n int(divmod(rem, 60)[1])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train Test Split
"},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"splitting into train and test data\"\"\"\ny = quora_train['target']\nX = quora_train.drop(columns = ['target'])\n\nX_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.20, stratify=y)\n#Printing the shape of train,cv and test dataset\nprint(\"The shape of train,cv & test dataset before conversion into vector\")\nprint(X_train.shape, y_train.shape)\nprint(X_cv.shape, y_cv.shape)\nprint(quora_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Vectorizing the text and numberical data
"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\nstart = time.time()\n\n#tokenization function\ndef tokenize(sentence): \n tokens = re.sub('[^a-zA-Z0-9]',\" \",sentence).split()\n return tokens\n\n#tfidf vectorizer\ntfidfvec = TfidfVectorizer(ngram_range=(1,3), min_df=5, max_df=0.9, strip_accents='unicode', \n tokenizer=tokenize,use_idf=True, smooth_idf=True, sublinear_tf=True)\n\nXtrain_text = tfidfvec.fit_transform(X_train['question_text_preprocessed'].values.astype(str))\nXcv_text = tfidfvec.transform(X_cv['question_text_preprocessed'].values.astype(str))\nXtest_text = tfidfvec.transform(quora_test['question_text_preprocessed'].values.astype(str))\n\nprint(\"Shape of matrix after one hot encoding:\")\nprint(Xtrain_text.shape, y_train.shape)\nprint(Xcv_text.shape, y_cv.shape)\nprint(Xtest_text.shape)\n\nhours, rem = divmod(time.time()-start, 3600)\nprint(\"Time to process the whole function : {:02}.{:0>2}.{:02} minutes.\".format(int(hours),\n int(divmod(rem, 60)[0]), \n int(divmod(rem, 60)[1])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.preprocessing import Normalizer\nnormalizer = Normalizer()\n#total_words\nnormalizer.fit(X_train['total_words'].values.reshape(-1,1))\nXtrain_total_words = normalizer.transform(X_train['total_words'].values.reshape(-1,1))\nXcv_total_words = normalizer.transform(X_cv['total_words'].values.reshape(-1,1))\nXtest_total_words = normalizer.transform(quora_test['total_words'].values.reshape(-1,1))\n\n#total_chars\nnormalizer.fit(X_train['total_chars'].values.reshape(-1,1))\nXtrain_total_chars = normalizer.transform(X_train['total_chars'].values.reshape(-1,1))\nXcv_total_chars = normalizer.transform(X_cv['total_chars'].values.reshape(-1,1))\nXtest_total_chars = normalizer.transform(quora_test['total_chars'].values.reshape(-1,1))\n\n#total_unique_words\nnormalizer.fit(X_train['total_unique_words'].values.reshape(-1,1))\nXtrain_total_unique_words = normalizer.transform(X_train['total_unique_words'].values.reshape(-1,1))\nXcv_total_unique_words = normalizer.transform(X_cv['total_unique_words'].values.reshape(-1,1))\nXtest_total_unique_words = normalizer.transform(quora_test['total_unique_words'].values.reshape(-1,1))\n\n#word_score\nnormalizer.fit(X_train['word_score'].values.reshape(-1,1))\nXtrain_word_score = normalizer.transform(X_train['word_score'].values.reshape(-1,1))\nXcv_word_score = normalizer.transform(X_cv['word_score'].values.reshape(-1,1))\nXtest_word_score = normalizer.transform(quora_test['word_score'].values.reshape(-1,1))\n\n#total_stopwords\nnormalizer.fit(X_train['total_stopwords'].values.reshape(-1,1))\nXtrain_total_stopwords = normalizer.transform(X_train['total_stopwords'].values.reshape(-1,1))\nXcv_total_stopwords = normalizer.transform(X_cv['total_stopwords'].values.reshape(-1,1))\nXtest_total_stopwords = normalizer.transform(quora_test['total_stopwords'].values.reshape(-1,1))\n\n#total_upper\nnormalizer.fit(X_train['total_upper'].values.reshape(-1,1))\nXtrain_total_upper = normalizer.transform(X_train['total_upper'].values.reshape(-1,1))\nXcv_total_upper = normalizer.transform(X_cv['total_upper'].values.reshape(-1,1))\nXtest_total_upper = normalizer.transform(quora_test['total_upper'].values.reshape(-1,1))\n\n#total_lower\nnormalizer.fit(X_train['total_lower'].values.reshape(-1,1))\nXtrain_total_lower = normalizer.transform(X_train['total_lower'].values.reshape(-1,1))\nXcv_total_lower = normalizer.transform(X_cv['total_lower'].values.reshape(-1,1))\nXtest_total_lower = normalizer.transform(quora_test['total_lower'].values.reshape(-1,1))\n\n#total_word_title\nnormalizer.fit(X_train['total_word_title'].values.reshape(-1,1))\nXtrain_total_word_title = normalizer.transform(X_train['total_word_title'].values.reshape(-1,1))\nXcv_total_word_title = normalizer.transform(X_cv['total_word_title'].values.reshape(-1,1))\nXtest_total_word_title = normalizer.transform(quora_test['total_word_title'].values.reshape(-1,1))\n\n#median_word_len\nnormalizer.fit(X_train['median_word_len'].values.reshape(-1,1))\nXtrain_median_word_len = normalizer.transform(X_train['median_word_len'].values.reshape(-1,1))\nXcv_median_word_len = normalizer.transform(X_cv['median_word_len'].values.reshape(-1,1))\nXtest_median_word_len = normalizer.transform(quora_test['median_word_len'].values.reshape(-1,1))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"stacking features
"},{"metadata":{"trusted":true},"cell_type":"code","source":"from scipy.sparse import hstack\n#stacking all features\nXtrain_quora = hstack((Xtrain_total_words, Xtrain_total_chars, Xtrain_total_unique_words, Xtrain_word_score,\n Xtrain_total_stopwords, Xtrain_total_upper, Xtrain_total_lower, Xtrain_total_word_title,\n Xtrain_median_word_len, Xtrain_text)).tocsr()\n\nXcv_quora = hstack((Xcv_total_words, Xcv_total_chars, Xcv_total_unique_words, Xcv_word_score,\n Xcv_total_stopwords, Xcv_total_upper, Xcv_total_lower, Xcv_total_word_title,\n Xcv_median_word_len, Xcv_text)).tocsr()\n\nXtest_quora =hstack((Xtest_total_words, Xtest_total_chars, Xtest_total_unique_words, Xtest_word_score,\n Xtest_total_stopwords, Xtest_total_upper, Xtest_total_lower, Xtest_total_word_title,\n Xtest_median_word_len, Xtest_text)).tocsr()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"ML Models
"},{"metadata":{},"cell_type":"markdown","source":"LightGBM
"},{"metadata":{"trusted":true},"cell_type":"code","source":"#https://lightgbm.readthedocs.io/en/latest/Parameters.html\nstart = time.time()\nimport lightgbm as lgbm\n#LGBMClassifier(boosting_type='gbdt', num_leaves=31, max_depth=- 1, learning_rate=0.1, n_estimators=100, subsample_for_bin=200000, \n#objective=None, class_weight=None, min_split_gain=0.0, min_child_weight=0.001, min_child_samples=20, subsample=1.0, subsample_freq=0, \n#colsample_bytree=1.0, reg_alpha=0.0, reg_lambda=0.0, random_state=None, n_jobs=- 1, silent=True, importance_type='split', **kwargs)\nlgb = lgbm.LGBMClassifier(boosting_type='gbdt', objective=\"binary\", metric=\"auc\", boost_from_average=False,max_depth=-1,\n learning_rate=0.3, max_bin=100, num_leaves=31, bagging_fraction = 0.8, feature_fraction = 0.8,\n scale_pos_weight = 1, num_threads = 32, verbosity = 0, n_jobs=- 1)\n\nvalues = [round(0.2 * x,2) for x in range(1, 6)]\nparam_grid = {'min_gain_to_split': values, 'lambda_l1':values, 'lambda_l2':values,\n \"n_estimators\":[100,200,500,1000]}\nclf = RandomizedSearchCV(lgb, param_grid, scoring='f1',return_train_score=True, verbose=5, n_jobs=-1)\nclf.fit(Xtrain_quora, y_train)\n\nprint(\"Best cross-validation score: {:.2f}\".format(clf.best_score_))\nprint(\"Best parameters: \", clf.best_params_)\n\n\nlgb = lgbm.LGBMClassifier(boosting_type='gbdt', objective=\"binary\", metric=\"auc\", boost_from_average=False, \n max_depth = -1,learning_rate=0.3, max_bin=100, num_leaves=31, bagging_fraction = 0.8, \n feature_fraction = 0.8,min_gain_to_split = clf.best_params_['min_gain_to_split'],\n lambda_l1 = clf.best_params_['lambda_l1'],lambda_l2 = clf.best_params_['lambda_l2'],\n n_estimators = clf.best_params_['n_estimators'], scale_pos_weight = 1, num_threads = 32,\n verbosity = 0)\nlgb.fit(Xtrain_quora,y_train)\ny_pred=lgb.predict(Xcv_quora)\n\nprint(\"----LightGBM----\")\nprint(\"Overall f1 score:\",round((metrics.f1_score(y_cv,y_pred)),2))\nprint(\"Overall Precision:\",round((metrics.precision_score(y_cv,y_pred)),2))\nprint(\"Overall Recall:\",round((metrics.recall_score(y_cv,y_pred)),2))\nprint(\"Classification Report:\\n\",metrics.classification_report(y_cv,y_pred))\n\nfig, ax = plot_confusion_matrix(conf_mat=metrics.confusion_matrix(y_cv,y_pred), figsize=(5, 5))\npt.show()\n\n#predicting output\nytestPred = lgb.predict(Xtest_quora)\nytestPred = (ytestPred>0.25).astype(int)\nquora_test = pd.DataFrame({'qid':quora_test['qid'].values})\nquora_test['prediction'] = ytestPred\nprint(\"Quora Test Output:\\n\",quora_test['prediction'].value_counts())\n\nhours, rem = divmod(time.time()-start, 3600)\nprint(\"Time to process the whole function : {:02}.{:0>2}.{:02} minutes.\".format(int(hours),\n int(divmod(rem, 60)[0]), \n int(divmod(rem, 60)[1])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Submitting values\nquora_test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}