{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Import Libraries
"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import io\nimport re\nimport os\nimport nltk\nimport time\nimport math\nimport scipy\nimport string\nimport zipfile\nimport operator\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.pyplot as pt\nfrom collections import defaultdict\nfrom gensim.models import KeyedVectors, Word2Vec, fasttext\nimport warnings\nimport zipfile\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Importing Data
"},{"metadata":{"trusted":true},"cell_type":"code","source":"quora_train = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/train.csv\")\nquora_test = pd.read_csv(\"/kaggle/input/quora-insincere-questions-classification/test.csv\")\nquora_train.head(1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Embedding Function
"},{"metadata":{"trusted":true},"cell_type":"code","source":"file_path = \"/kaggle/input/quora-insincere-questions-classification/embeddings.zip\"\n\ndef Embeddings(file_path,file):\n '''\n parameter : file_path(embedding file), \n file = name of the file\n return : embedding_matrix(dictionary)\n ''' \n embeddings_glove = dict()\n with zipfile.ZipFile(file_path,'r') as zf:\n if file == \"glove\":\n with io.TextIOWrapper(zf.open(\"glove.840B.300d/glove.840B.300d.txt\"), encoding=\"utf-8\") as f:\n for line in f:\n values=line.split(' ') # \".split(' ')\" only for glove-840b-300d; for all other files, \".split()\" works\n word=values[0]\n vectors=np.asarray(values[1:],'float32')\n embeddings_glove[word]=vectors\n return embeddings_glove\n \n elif file == \"word2vec\":\n embeddings_glove = KeyedVectors.load_word2vec_format(zf.open(\"GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\"),binary=True)\n return embeddings_glove\n\n elif file == \"paragram\":\n path = zf.extract(\"paragram_300_sl999/paragram_300_sl999.txt\")\n def get_coefs(word,*arr): \n return word, np.asarray(arr, dtype='float32')\n embeddings_glove = dict(get_coefs(*w.split(\" \")) for w in open(path, encoding='latin'))\n return embeddings_glove\n\n elif file==\"fasttext\":\n path = zf.extract(\"wiki-news-300d-1M/wiki-news-300d-1M.vec\")\n def get_coefs(word,*arr): \n return word, np.asarray(arr, dtype='float32')\n embeddings_glove = dict(get_coefs(*w.split(\" \")) for w in open(path, encoding='latin'))\n return embeddings_glove","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"paragramModel = Embeddings(file_path,file=\"paragram\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Text Preprocessing
"},{"metadata":{"trusted":true},"cell_type":"code","source":"def str_lowercase_text(text):\n #mapping all text into str and lowercase\n text = text.map(str)\n #convert all the words to lower case first and then remove the stopwords\n for line in range(len(text.values)):\n text.values[line] = text.values[line].lower()\n return text\nquora_train['question_text_paragram'] = str_lowercase_text(quora_train['question_text'])\nquora_test['question_text_paragram'] = str_lowercase_text(quora_test['question_text'])\nprint(\"str_lowercase_text : Done\")\n\n#dictionary of special characters and their literal meaning to replace in the vocabulary\nreplacingWords = dict({\"√\":\" sqrt \",\"π\":\" pi \",\"α\":\" alpha \",\"θ\":\" theta \",\"∞\":\" infinity \",\n\"∝\":\" proportional to \",\"sinx\":\" sin x \",\"cosx\":\" cos x \", \"tanx\":\" tan x \",\"cotx\":\" cot x \", \n\"secx\":\" sec x \", \"cosecx\":\" cosec x \", \"£\":\" pound \", \"β\":\" beta \", \"σ\": \" theta \", \"∆\":\" delta \",\n\"μ\":\" mu \",'∫': \" integration \", \"ρ\":\" rho \", \"λ\":\" lambda \",\"∩\":\" intersection \",\"Δ\":\" delta \", \n\"φ\":\" phi \", \"℃\":\" centigrade \",\"≠\":\" does not equal to \",\"Ω\":\" omega \",\"∑\":\" summation \",\n\"∪\":\" union \", \"ψ\":\" psi \", \"Γ\":\" gamma \",\"⇒\":\" implies \",\"∈\":\" is an element of \", \n\"≡\":\" is congruent to \", \"≈\":\" is approximately equal to \", \"~\":\" is distributed as \",\n\"≅\":\" is isomorphic to \",\"⩽\":\" is less than or equal to \",\"≥\":\" is greater than or equal to \",\n\"⇐\":\" is implied by \",\"⇔\":\" is equivalent to \", \"∉\":\" is not an element of \",\"∅\" : \" empty set \",\n\"∛\":\"cube root\",\"÷\":\" division \",\"㏒\":\" log \",\"∇\":\" del \",\"⊆\":\" is a subset of \",\"±\":\" plus–minus \",\n\"⊂\":\" is a proper subset of \",\"€\":\" euro \",\"㏑\":\" ln \",\"₹\":\" rupee \",\"∀\":\" there exists \",\"∛\":\"cube root\",\n\"⅓\":\" one by three \",\"½\":\" one by two \",\"∈\":\" is an element of \",\"¼\":\" one by four \"})\n\ndef special_chars(text,symbols):\n for p in symbols:\n if p in text:\n text = text.replace(p, symbols[p])\n text = re.sub(\"\\s{2}\",\" \",text)\n return text\nquora_train['question_text_paragram'] = quora_train['question_text_paragram'].apply(lambda x: special_chars(x,replacingWords))\nquora_test['question_text_paragram'] = quora_test['question_text_paragram'].apply(lambda x: special_chars(x,replacingWords))\nprint(\"special_chars : Done\")\n\n#characters to be removed\ncharacters = \"＝ా̫̾̀ͅ⚧ਿ∖્⬇☉ూिାੁ͔☺\u0017͛ॢി「̷َ̊̆«̰︡？◦✏ូ͒\u0013ِ℅„〖ௌ•‐̗∧̯িֿ̔〗்“·้″∂͚̑ी∴ు́̕♡\u0001¯❓̦̓ை₊ு\u0002ं”☁×ొ、್⌚—’̶̋̐⎝ैು¸̞͑（⋅ृ′͋‘়͊➡†ী️ா̥：̻ू∗＾\\\n\u001b→´្ाাோ✌。⊥̵̛¬̒ാ–»！∨・❤̝ంి̮⎞」͆।☝่̙˚̬͌͘¦്，̴̂˜ੀ│ីৃ⃗᠌¡ੰ̧♀✔̈́̓）⊨✅⎛\u001aិ\u0010↓्̉ुॣ；ّุ∡̭∘－؟△⋯ॉॄ✓∠̲̺®♭̱̍ു్̹̌̚͜ौ⁻ె⟨͈́⎠ँ™͝ំಿ\u0003¢̿ં͠↑،ើ̪ਂ₩̄̎‑̢ಾ¾₱̃ো︠ਾ≱͖ः¨ાி͡ে਼ៃ̣͂》♏̜\\\n̜§͕《͎≤̇〇／ْ₦̼¥▾…ះี̖̘͇̽͐ា‛♨‰़̤̳̅⧽−ೋ̀▒ٌู̩⅔♣̡ั∼͉¿͗\u0006°☹⟩্©¶ो˂＞े◌̸ీ⧼＄̟̈⦁\"\n\n#characters to be retained and spaced between words\ncharacter_list = ['^', ')', '@', ',', '$', '+', '/', '?', '\"', ';', '[', '%', '*', ']', \"'\", '>', '|', '=', '<', '.',\n '&', '`', '\\\\', '#', '}', '-', '!', ':', '{', '(', '_']\n\n#removing punctuation\ndef remove_punct(text,punToBeRemove):\n translate_table = dict((ord(char), None) for char in punToBeRemove) \n #Loop to iterate \n for idx,val in (enumerate(text.values)):\n val = val.translate(translate_table)\n text.values[idx] = val.strip()\n return text\n \nquora_train['question_text_paragram'] = remove_punct(quora_train['question_text_paragram'], characters)\nquora_test['question_text_paragram'] = remove_punct(quora_test['question_text_paragram'], characters) \nprint(\"remove_punct : Done\")\n\n#Source: - https://en.wikipedia.org/wiki/Wikipedia:List_of_English_contractions\ncontractions = {\"'aight\": 'alright', \"ain't\": 'am not', \"amn't\": 'am not', \"aren't\": 'are not', \"can't\": 'can not',\n\"'cause\": 'because', \"could've\": 'could have', \"couldn't\": 'could not', \"couldn't've\": 'could not have', \"daren't\": \n'dare not', \"daresn't\": 'dare not', \"dasn't\": 'dare not', \"didn't\": 'did not', \"doesn't\": 'does not', \n\"don't\": 'do not', 'dunno': \"don't know\", \"d'ye\": 'do you', \"e'er\": 'ever', \"everybody's\": 'everybody is', \n\"everyone's\": 'everyone is', 'finna': 'fixing to', \"g'day\": 'good day', 'gimme': 'give me', \"giv'n\": 'given', \n'gonna': 'going to', \"gon't\": 'go not', 'gotta': 'got to', \"hadn't\": 'had not', \"had've\": 'had have', \n\"hasn't\": 'has not', \"haven't\": 'have not', \"he'd\": 'he had', \"he'll\": 'he will', \"he's\": 'he is', \n\"he've\": 'he have', \"how'd\": 'how did', 'howdy': 'how do you do', \"how'll\": 'how will', \"how're\": 'how are', \n\"how's\": 'how is', \"I'd\": 'I had', \"I'd've\": 'I would have', \"I'll\": 'I will', \"I'm\": 'I am', \n\"I'm'a\": 'I am about to', \"I'm'o\": 'I am going to', 'innit': 'is it not', \"I've\": 'I have', \"isn't\": 'is not', \n\"it'd \": 'it would', \"it'll\": 'it will', \"it's \": 'it is', 'iunno': \"I don't know\", \"let's\": 'let us', \n\"ma'am\": 'madam', \"mayn't\": 'may not', \"may've\": 'may have', 'methinks': 'me thinks', \"mightn't\": 'might not', \n\"might've\": 'might have', \"mustn't\": 'must not', \"mustn't've\": 'must not have', \"must've\": 'must have', \n\"needn't\": 'need not', 'nal': 'and all', \"ne'er\": 'never', \"o'clock\": 'of the clock', \"o'er\": 'over',\n\"ol'\": 'old', \"oughtn't\": 'ought not', \"'s\": 'is', \"shalln't\": 'shall not', \"shan't\": 'shall not', \n\"she'd\": 'she would', \"she'll\": 'she will', \"she's\": 'she is', \"should've\": 'should have', \n\"shouldn't\": 'should not', \"shouldn't've\": 'should not have', \"somebody's\": 'somebody has', \n\"someone's\": 'someone has', \"something's\": 'something has', \"so're\": 'so are', \"that'll\": 'that will', \n\"that're\": 'that are', \"that's\": 'that is', \"that'd\": 'that would', \"there'd\": 'there would', \n\"there'll\": 'there will', \"there're\": 'there are', \"there's\": 'there is', \"these're\": 'these are', \n\"they've\": 'they have', \"this's\": 'this is', \"those're\": 'those are', \"those've\": 'those have', \"'tis\": 'it is', \n\"to've\": 'to have', \"'twas\": 'it was', 'wanna': 'want to', \"wasn't\": 'was not', \"we'd\": 'we would', \n\"we'd've\": 'we would have', \"we'll\": 'we will', \"we're\": 'we are', \"we've\": 'we have', \"weren't\": 'were not', \n\"what'd\": 'what did', \"what'll\": 'what will', \"what're\": 'what are', \"what's\": 'what does', \"what've\": 'what have',\n\"when's\": 'when is', \"where'd\": 'where did', \"where'll\": 'where will', \"where're\": 'where are',\n\"where's\": 'where is',\"where've\": 'where have', \"which'd\": 'which would', \"which'll\": 'which will', \n\"which're\": 'which are',\"which's\": 'which is', \"which've\": 'which have', \"who'd\": 'who would',\n\"who'd've\": 'who would have', \"who'll\": 'who will', \"who're\": 'who are', \"who'ves\": 'who is', \"who'\": 'who have',\n\"why'd\": 'why did', \"why're\": 'why are', \"why's\": 'why does', \"willn't\": 'will not', \"won't\": 'will not',\n'wonnot': 'will not', \"would've\": 'would have', \"wouldn't\": 'would not', \"wouldn't've\": 'would not have',\n\"y'all\": 'you all', \"y'all'd've\": 'you all would have', \"y'all'd'n've\": 'you all would not have',\n\"y'all're\": 'you all are', \"cause\":\"because\",\"have't\":\"have not\",\"cann't\":\"can not\",\"ain't\":\"am not\",\n\"you'd\": 'you would', \"you'll\": 'you will', \"you're\": 'you are', \"you've\": 'you have', 'cannot': 'can not', \n'wont': 'will not', \"You'\": 'Am not', \"Ain'\": 'Am not', \"Amn'\": 'Am not', \"Aren'\": 'Are not',\n\"Can'\": 'Because', \"Could'\": 'Could have', \"Couldn'\": 'Could not have', \"Daren'\": 'Dare not', \n\"Daresn'\": 'Dare not', \"Dasn'\": 'Dare not', \"Didn'\": 'Did not', \"Doesn'\": 'Does not', \"Don'\": \"Don't know\", \n\"D'\": 'Do you', \"E'\": 'Ever', \"Everybody'\": 'Everybody is', \"Everyone'\": 'Fixing to', \"G'\": 'Give me', \n\"Giv'\": 'Going to', \"Gon'\": 'Got to', \"Hadn'\": 'Had not', \"Had'\": 'Had have', \"Hasn'\": 'Has not', \n\"Haven'\": 'Have not', \"He'\": 'He have', \"How'\": 'How is', \"I'\": 'I have', \"Isn'\": 'Is not', \"It'\": \"I don't know\", \n\"Let'\": 'Let us', \"Ma'\": 'Madam', \"Mayn'\": 'May not', \"May'\": 'Me thinks', \"Mightn'\": 'Might not', \n\"Might'\": 'Might have', \"Mustn'\": 'Must not have', \"Must'\": 'Must have', \"Needn'\": 'And all', \"Ne'\": 'Never',\n\"O'\": 'Old', \"Oughtn'\": 'Is', \"Shalln'\": 'Shall not', \"Shan'\": 'Shall not', \"She'\": 'She is', \n\"Should'\": 'Should have', \"Shouldn'\": 'Should not have', \"Somebody'\": 'Somebody has', \"Someone'\": 'Someone has', \n\"Something'\": 'Something has', \"So'\": 'So are', \"That'\": 'That would', \"There'\": 'There is',\n\"They'\": 'They have', \"This'\": 'This is', \"Those'\": 'It is', \"To'\": 'Want to', \"Wasn'\": 'Was not',\n\"Weren'\": 'Were not', \"What'\": 'What have', \"When'\": 'When is', \"Where'\": 'Where have', \"Which'\": 'Which have', \n\"Who'\": 'Who have', \"Why'\": 'Why does', \"Willn'\": 'Will not', \"Won'\": 'Will not', \"Would'\": 'Would have',\n\"Wouldn'\": 'Would not have', \"Y'\": 'You all are',\"What's\":\"What is\",\"What're\":\"What are\",\"what's\":\"what is\",\n\"what're\":\"what are\", \"Who're\":\"Who are\", \"your're\":\"you are\",\"you're\":\"you are\", \"You're\":\"You are\",\n\"We're\":\"We are\", \"These'\": 'These have', \"we're\":\"we are\",\"Why're\":\"Why are\",\"How're\":\"How are \",\n\"how're \":\"how are \",\"they're \":\"they are \", \"befo're\":\"before\",\"'re \":\" are \",'don\"t ':\"do not\", \n\"Won't \":\"Will not \",\"could't\":\"could not\", \"would't\":\"would not\", \"We'\": 'We have',\"Hasn't\":\"Has not\",\n\"n't\":\"not\", 'who\"s':\"who is\"}\n\n#function to remove contractions\ndef decontraction(text,contractions):\n #Loop to iterate \n for idx,val in enumerate(text.values):\n val = ' '.join(word.replace(word,contractions[word]) if word in contractions\n else word for word in val.split())\n #generic one\n val = re.sub(r\"\\'s\", \" \", val);val = re.sub(r\"\\''s\", \" \", val);val = re.sub(r\"\\\"s\", \" \", val)\n val = re.sub(r\"n\\'t\", \" not \", val);val = re.sub(r\"n\\''t\", \" not \", val);val = re.sub(r\"n\\\"t\", \" not \", val)\n val = re.sub(r\"\\'re \", \" are \", val);val = re.sub(r\"\\'d \", \" would\", val);val = re.sub(r\"\\''d \", \" would\", val)\n val = re.sub(r\"\\\"d \", \" would\", val);val = re.sub(r\"\\'ll \", \" will\", val);val = re.sub(r\"\\''ll \", \" will\", val)\n val = re.sub(r\"\\\"ll \", \" will\", val);val = re.sub(r\"\\'ve \", \" have\", val);val = re.sub(r\"\\''ve \", \" have\", val)\n val = re.sub(r\"\\\"ve \", \" have\", val);val = re.sub(r\"\\'m \", \" am\", val);val = re.sub(r\"\\''m \",\" am\", val)\n val = re.sub(r\"\\\"m \",\" am\", val);val = re.sub(\"\\s{2}\",\" \",val)\n text.values[idx] = val.strip() \n return text\n\n\"\"\"function to replace special characters with their respective meanings\"\"\"\ndef spacing_of_chars(text,characters_list):\n for char in characters_list:\n if char in text:\n text = text.replace(char,\" \"+char+\" \")\n text = re.sub(\"\\s+\",\" \",text)\n return text\n\nquora_train['question_text_paragram'] = quora_train['question_text_paragram'].apply(lambda x: spacing_of_chars(x,character_list))\nquora_test['question_text_paragram'] = quora_test['question_text_paragram'].apply(lambda x: spacing_of_chars(x,character_list))\nprint(\"spacing_of_chars : Done\")\n\n\n#checking coverage for words present in question_text and in embedding_matrix\ndef coverage(vocab, embeddings_index,print_statement=False):\n #Initializing values\n known_words = defaultdict(int)\n unknown_words = defaultdict(int)\n knownWordsVal = 0\n unknownWordsVal = 0\n #iterating words\n for word in vocab.keys():\n try:\n known_words[word] = embeddings_index[word]\n knownWordsVal += vocab[word]\n except:\n unknown_words[word] = vocab[word]\n unknownWordsVal += vocab[word]\n pass\n \n if print_statement == True:\n print('Found {:.2%} of words in the embedding of the question text vocab'\n .format(len(known_words) / len(vocab)))\n print('Found {:.2%} of the words in the question text vocab'.format(knownWordsVal / (knownWordsVal + unknownWordsVal)))\n else:\n pass\n unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n return unknown_words\n\n\n#function to generate vocabulary of question text\ndef question_text_vocab(text):\n freq_dict = defaultdict(int)\n total_sent = text.apply(lambda x: x.split()).values\n for sent in total_sent:\n for token in sent:\n freq_dict[token] += 1\n return freq_dict\n\n\n#question text vocab\nTextVocabTrain = question_text_vocab(quora_train['question_text_paragram'])\nTextVocabTest = question_text_vocab(quora_test['question_text_paragram'])\nprint(\"question_text_vocab : Done\")\n\n\n#coverage function\nOOVGloveTrain = coverage(TextVocabTrain, paragramModel)\nOOVGloveTest = coverage(TextVocabTest, paragramModel)\nprint(\"coverage : Done\")\n\n#checking for the words that are present in the embeddings with lowercase or as title\ndef check_oov_vocab(vocab, embeddings):\n freq_dict = defaultdict()\n for word in vocab:\n if word[0].istitle() == True:\n if word[0].lower() in embeddings:\n freq_dict[word[0]]= word[0].lower()\n elif word[0].islower() == True:\n if word[0].title() in embeddings:\n freq_dict[word[0]]= word[0].title()\n return freq_dict\n\nWordDictTrain = check_oov_vocab(OOVGloveTrain,paragramModel)\nWordDictTest = check_oov_vocab(OOVGloveTest,paragramModel)\nprint(\"check_oov_vocab : Done\")\n\ndef replace_words(text,set_words):\n for idx,val in enumerate(text.values):\n val = ' '.join(word.replace(word,set_words[word]) if word in set_words else word for word in val.split())\n text.values[idx] = val\n return text\n\nquora_train['question_text_paragram'] = replace_words(quora_train['question_text_paragram'], WordDictTrain)\nquora_test['question_text_paragram'] = replace_words(quora_test['question_text_paragram'], WordDictTest)\nprint(\"replace_words : Done\")\n\nreplace_word = dict({\"quorans\":\"quora\", \"brexit\":\"britain exit\", \"cryptocurrencies\":\"cryptocurrency\", \n\"blockchain\":\"blockchain\", \"demonetisation\":\"demonetization\", \"pokémon\":\"pokemon\",\n\"qoura\":\"quora\", \"fiancé\":\"fiance\",\"cryptocurrency\":\"cryptocurrency\", \"x²\":\"x squeare\", \n\"quoras\":\"quora\",\"whst\":\"what\", \"²\":\"square\", \"Demonetization\":\"demonetization\", \n\"brexit\":\"britain exit\", \"são\":\"sao\",\"genderfluid\":\"Gender fluid\", \"howcan\":\"How can\", \n\"undergraduation\":\"under graduation\", \"whydo\":\"why do\", \"à\":\"a\",\"chapterwise\":\"chapter wise\",\n\"cryptocurrencies\":\"cryptocurrency\", \"fiancée\":\"fiance\", \"wouldwin\":\"would win\", \n\"nanodegree\":\"nano degree\",\"nanodegree\":\"nano degree\", \"blockchains\":\"blockchain\", \n\"clichés\":\"cliche\", \"erdoğan\":\"erdogan\", \"beyoncé\":\"beyonce\", \"fullform\":\"full form\",\n\"atatürk\":\"ataturk\", \"Whyis\":\"Why is\",\"amfrom\":\"am from\", \"2k17\":\"2017\", \n\"demonitization\":\"demonetization\", \"cliché\":\"cliche\", \"montréal\":\"montreal\", \n\"thé\":\"the\", \"am17\":\"am 17\", \"willhappen\":\"will happen\",\"³\":\"cube\", \"whatapp\":\"whatsapp\", \n\"ε\":\"epsilon\", \"whatsaap\":\"whatsapp\",'Σ':\"summation\",\"quorians\":\"quora users\",\n\"cryptocurreny\":\"cryptocurrency\", \"mastuburation\":\"masturbation\",\"Whatre\":\"What are\", \n\"whatdo\":\"what do\",\"δ\":\"delta\",\"oversmart\":\"over smart\",\"¹\":\"one\",\"baahubali\":\"baahubali\", \n\"note4\":\"note 4\", \"gdpr\":\"general data protection regulation\", \"bnbr\":\"' be nice , be respectful '\", \n\"uceed\":\"undergraduate common entrance examination for design\",\"bhakts\":\"bhakts\", \n\"iiest\":\"indian institutes of engineering science and technology\",\"bhakths\":\"bhakts\",\n\"upwork\":\"Upwork\",\"unacademy\":\"Unacademy\",\"squeare\":\"square\",\"srmjeee\":\"srmjee\",\n\"demonitisation\":\"demonetization\",\n\"cos2x\":\"cos 2x\",\"padmavat\":\"padmaavat\", \"flipcart\":\"flipkart\",\n\"havegot\":\"have got\",\"2k18\":\"2018\",\"a²\":\"a square\",\"whydoes\":\"why does\",\"sina\":\"sin a\",\n\"class9\":\"class 9\"})\n\n#replacing words with correct once\nquora_train['question_text_paragram'] = replace_words(quora_train['question_text_paragram'], replace_word)\nquora_test['question_text_paragram'] = replace_words(quora_test['question_text_paragram'], replace_word)\nprint(\"replace_words : Done\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Tensorflow Dependencies
"},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.keras import backend as K\nfrom tensorflow.keras.backend import clear_session, maximum\nfrom tensorflow.keras.callbacks import EarlyStopping \nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.layers import Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom tensorflow.keras import Model, initializers, regularizers, constraints, optimizers, layers\nfrom tensorflow.keras.regularizers import l2\nfrom tensorflow.keras.layers import Dense, Input, Dropout, Conv1D, MaxPooling1D, Flatten, Embedding\nfrom tensorflow.keras.layers import Concatenate, LSTM, Activation, GRU, Reshape, Lambda, Multiply, Bidirectional, Maximum\nfrom tensorflow.keras.preprocessing.text import Tokenizer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Splitting of data
"},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\ny = quora_train['target']\nX = quora_train.drop(columns = ['target'])\n\nX_train, X_cv, y_train, y_cv = train_test_split(X, y, test_size=0.20, stratify=y)\nprint(\"The shape of train,cv & test dataset before conversion into vector\")\nprint(X_train.shape, y_train.shape)\nprint(X_cv.shape, y_cv.shape)\nprint(quora_test.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Tokenization and Sequence Padding
"},{"metadata":{"trusted":true},"cell_type":"code","source":"#define values\nmaxlength = 75\nembedding_dim = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def TokenizationPadding(data, maxlen): #pass data in list [X_train[\"\"],X_cv[\"\"],X_test[\"\"]]\n \n #tokenizing dataset\n encoder_data = list()\n tokens = Tokenizer()\n tokens.fit_on_texts(data[0])\n for idx,val in enumerate(data):\n encoder_data.append(tokens.texts_to_sequences(val))\n \n #vocab_size\n vocab_size = len(tokens.word_index)+1\n \n #sequence padding\n seq_padding = list()\n for val in encoder_data:\n seq_padding.append(pad_sequences(val, maxlen=maxlen, padding='post'))\n \n return seq_padding, vocab_size, tokens","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data = [X_train[\"question_text_paragram\"], X_cv[\"question_text_paragram\"], quora_test[\"question_text_paragram\"]]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"seq_padding, vocab_size, tokenizer = TokenizationPadding(data, maxlength)\nXtrain, Xcv, Xtest = seq_padding[0], seq_padding[1], seq_padding[2]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Embedding Matrix
"},{"metadata":{"trusted":true},"cell_type":"code","source":"def embedding_matrix(vocab_size,model,dim,tokenizer):\n #getting embedding matrix\n keys = set(model.keys())\n emb_matrix = np.zeros((vocab_size,dim))\n for idx,val in tokenizer.word_index.items():\n if idx in keys:\n #vector\n emb_vector = model[idx]\n #matrix\n emb_matrix[val] = emb_vector\n \n print('The shape of emdedding matrix is: ',emb_matrix.shape)\n return emb_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix_paragram = embedding_matrix(vocab_size, paragramModel, embedding_dim, tokenizer)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Converting output to categorical data
"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.utils import to_categorical\nytrain = to_categorical(y_train, 2)\nycv = to_categorical(y_cv, 2)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Callbacks
"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Callback function\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn.metrics import confusion_matrix, f1_score\nfrom tensorflow.keras.callbacks import Callback\n\nclass accuracy_value(Callback):\n\n def __init__(self,training_data,validation_data):\n self.X_train = training_data[0]\n self.y_train = training_data[1]\n self.X_val = validation_data[0]\n self.y_val = validation_data[1]\n\n def on_train_begin(self, logs = {}):\n self.f1_scores = []\n self.precisions = []\n self.recalls = []\n\n def on_epoch_end(self, epoch, logs = {}):\n #F1 Score\n y_predicted = np.asarray(self.model.predict(self.X_val)).round()\n f1_val = f1_score(self.y_val,y_predicted,average=None)\n self.f1_scores.append(f1_val)\n\n print(\" - f1 score : {}\".format(np.round(f1_val,4)))\n\nf1Score = accuracy_value(training_data=(Xtrain, ytrain), validation_data=(Xcv, ycv))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"earlyStopping = EarlyStopping(monitor='val_loss', min_delta=0, patience=0, verbose=0, mode='auto')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"LSTM with Bahdanau Attention Model
"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(tf.keras.layers.Layer):\n '''\n Class the calculates score based on the scoring_function using Bahdanu attention mechanism.\n '''\n def __init__(self, att_units):\n super(Attention, self).__init__()\n #number of attention units to be provided\n self.att_units = att_units\n \n # Intialize variables needed for Concat score function here\n self.W1=tf.keras.layers.Dense(att_units)\n self.W2=tf.keras.layers.Dense(att_units)\n self.V=tf.keras.layers.Dense(1)\n\n \n def call(self,lstm_output, hidden_state):\n \n \"\"\"\n hidden state shape == (batch_size, hidden size)\n lstm output shape == (batch_size, max_length, hidden size)\n \"\"\"\n \n #state_with_time_axis shape == (batch_size, 1, hidden size)\n #we are doing this to broadcast addition along the time axis to calculate the score\n state_with_time_axis = tf.expand_dims(hidden_state, 1)\n \n #score shape == (batch_size, max_length, 1)\n #we get 1 at the last axis because we are applying score to self.V\n #the shape of the tensor before applying self.V is (batch_size, max_length, units)\n score = self.V(tf.nn.tanh(self.W1(state_with_time_axis) + self.W2(lstm_output)))\n\n # attention_weights shape == (batch_size, max_length, 1)\n attention_weights=tf.nn.softmax(score, axis=1)\n\n # context_vector shape after sum == (batch_size, hidden_size)\n context_vector = attention_weights*lstm_output\n context_vector = tf.reduce_sum(context_vector, axis=1) \n\n return context_vector, attention_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clear_session()\n#input\ninputs = Input(shape=(maxlength,), dtype='int32', name='Input_Text')\n#embedding layer\nEmbedding_Layer = Embedding(vocab_size, 300, weights=[embedding_matrix_paragram], input_length=maxlength, trainable=False)(inputs)\n#bidirectional lstm with cell and hidden state \nlstm_output, fw_state_h, fw_state_c, bw_state_h, bw_state_c = Bidirectional(LSTM(64, return_sequences=True, return_state=True))(Embedding_Layer)\nstate_h = Concatenate()([fw_state_h, bw_state_h])\n#getting context vector from attention model\ncontext_vector, attention_weights = Attention(10)(lstm_output, state_h)\n#dense layer\ndense = Dense(64, activation='relu', name= \"Dense_Layer\")(context_vector)\noutput = Dense(2, activation='sigmoid', name= \"Output_Layer\")(dense)\nmodel = Model(inputs,output)\n\n#compiling model\nmodel.compile(loss = \"binary_crossentropy\", optimizer = \"adam\", metrics = [\"accuracy\"]) #compile the model\n#fitting the model\nmodel.fit(Xtrain, ytrain, batch_size=512, verbose=1, epochs=10,validation_data=(Xcv,ycv), shuffle=True, callbacks=[f1Score, earlyStopping])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"threshold = dict()\nypred = model.predict(Xcv, batch_size=512,verbose=1)\nfrom sklearn import metrics\nfor thresh in np.arange(0.1, 0.501, 0.05):\n thresh = np.round(thresh, 2)\n print(\"F1 score at threshold {0} is {1}\".format(thresh, round(list(metrics.f1_score(ycv, (ypred>thresh).astype(int), average=None))[1],3)))\n threshold[thresh] = round(list(metrics.f1_score(ycv, (ypred>thresh).astype(int), average=None))[1],3)\n\nprint(\"\\nThe best threshold is:\",max(threshold, key=threshold.get))\n\n#printing classification report using the best threshold\nypredicted = (ypred>max(threshold, key=threshold.get)).astype(int)\nprint(\"Classification Report:\\n\",metrics.classification_report(ycv,ypredicted))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#predicting test data\nypredict = list()\nypred = model.predict(Xtest, batch_size=512,verbose=1)\nfor i in ypred:\n ypredict.append((i[1]>max(threshold, key=threshold.get)).astype(int))\n\n#creating dataframe\ndf_test = pd.DataFrame({\"qid\":quora_test[\"qid\"].values})\ndf_test['prediction'] = ypredict\nprint(\"Quora Test Output:\\n\",df_test['prediction'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}