{"cells":[{"metadata":{"_uuid":"b94b3233b3d7296b4deb13b072d5691c91ba71ca"},"cell_type":"markdown","source":"# Import Libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)maxle\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d71476a3919615a3205a883377b9e56a00172470"},"cell_type":"markdown","source":"print(os.listdir(\"../input/embeddings/wiki-news-300d-1M\"))"},{"metadata":{"trusted":true,"_uuid":"fcfe613ecf6d95ccf11d6d1817269e540550b6e1"},"cell_type":"code","source":"import pandas as pd\nimport tensorflow as tf\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport keras\nimport operator ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"75775dd1ec92ccf606fc344c051c6d739cbccd05"},"cell_type":"markdown","source":"# Import Data"},{"metadata":{"trusted":true,"_uuid":"1792d32e9de24f66f4c3fb79358d493ef6338b2d"},"cell_type":"code","source":"df_train=pd.read_csv('../input/train.csv')\ndf_test=pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"887e1721bfb98cb69f1c56dd5a5c3ce0b92e5ff4"},"cell_type":"markdown","source":"df_train\n\n                        qid\t       question_text\t                       target\tlength\n0\t00002165364db923c7e6\tHow did Quebec nationalists see their province...\t       0\t          13\n1\t000032939017120e6e44\tDo you have an adopted dog, how would you enco...\t 0\t            16\n2\t0000412ca6e4628ce2cf\tWhy does velocity affect time? Does velocity a...\t         0\t            10\n3\t000042bf85aa498cd78e\tHow did Otto von Guericke used the Magdeburg h...\t  0             \t9\n4\t0000455dfa3e01eae3af\tCan I convert montra helicon D to a mountain b...\t       0\t            15\n5\t00004f9a462a357c33be\tIs Gaza slowly becoming Auschwitz, Dachau or T...\t   0\t            10\n6\t00005059a06ee19e11ad\tWhy does Quora automatically ban conservative ...\t  0\t                18\n7\t0000559f875832745e2e\tIs it crazy if I wash or wipe my groceries off...\t            0\t              14\n8\t00005bd3426b2d0c8305\tIs there such a thing as dressing moderately, ...\t        0\t               18\n9\t00006e6928c5df60eacb\tIs it just me or have you ever been in this ph...\t           0\t             44\n10\t000075f67dd595c3deb5\tWhat can you say about feminism?\t                         0\t                 6"},{"metadata":{"_uuid":"1f732ec3c2d62b6f56d16720ac3a043fe4bd7af4"},"cell_type":"markdown","source":"# class distribution"},{"metadata":{"trusted":true,"_uuid":"127c8dbe50a9c0c97c44e915a2a9dd9726b3eeb7"},"cell_type":"code","source":"print (\"Train data target 1\")\nprint (df_train[df_train['target']==1].count())\n\nprint (\"Train data target 0\")\nprint (df_train[df_train['target']==0].count())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"be4a7c983d66492e8ca4abec9d7188dcf52254d9"},"cell_type":"markdown","source":"# All text - word cloud"},{"metadata":{"trusted":true,"_uuid":"3e75f731c8912ddf32090bba548b7f8384bcca0f"},"cell_type":"code","source":"all_phrases=df_train[df_train.target != 2]\nall_words = []\nfor t in all_phrases.question_text:\n    all_words.append(t)\nall_words[:4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b0cbe8e7100803e7e42e4a3a9e5cbb2028a81643"},"cell_type":"code","source":"all_text = pd.Series(all_words).str.cat(sep=' ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa42ed89ceb03ec153fd23867e28045db05664a3"},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nwordcloud = WordCloud(width=1600, height=800, max_font_size=200).generate(all_text)\nplt.figure(figsize=(12,10))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"38889ee8393a0b4b1b45f476cd96034de64af450"},"cell_type":"markdown","source":"# Insincere Questions - word cloud"},{"metadata":{"trusted":true,"_uuid":"e0e2a56ae6b616a43202c0ef299b9ba386452d8b"},"cell_type":"code","source":"neg_phrases = df_train[df_train.target == 1]\nneg_words = []\nfor t in neg_phrases.question_text:\n    neg_words.append(t)\nneg_words[:4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25501f295b3e923d75e76cf352414eb3ebe8579b"},"cell_type":"code","source":"neg_text = pd.Series(neg_words).str.cat(sep=' ')\nneg_text[:100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3e9fb606faeace0e255fa828aa1439be20e886d9"},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nwordcloud = WordCloud(width=1600, height=800, max_font_size=200).generate(neg_text)\nplt.figure(figsize=(12,10))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4f65f18559d9200636bec9602f16b40ebca10aeb"},"cell_type":"markdown","source":"# Sincere Questions - word cloud"},{"metadata":{"trusted":true,"_uuid":"a264b2a89f343f1c548e74dd09fba8c6f30388d4"},"cell_type":"code","source":"pos_phrases = df_train[df_train.target == 0]\npos_words = []\nfor t in pos_phrases.question_text:\n    pos_words.append(t)\npos_words[:4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e0516ca683687fb22ae5e4a7bd88fa921855e0ae"},"cell_type":"code","source":"pos_text = pd.Series(pos_words).str.cat(sep=' ')\npos_text[:100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e7cb064b199b68493ccbe8d0e2e531503528a4da"},"cell_type":"code","source":"from wordcloud import WordCloud\nwordcloud = WordCloud(width=1600, height=800, max_font_size=200).generate(pos_text)\nplt.figure(figsize=(12,10))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd06e1224b0b6c2b582efa65c1c88ebd1fba152b"},"cell_type":"markdown","source":"# Add new feature : length of the sentence"},{"metadata":{"trusted":true,"_uuid":"968cf51093d7b5b65b29a06465d9e829013ab8da"},"cell_type":"code","source":"df_train['length'] = df_train['question_text'].str.count(' ') + 1\ndf_test['length'] = df_test['question_text'].str.count(' ') + 1","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"23ae47bb39ab521b013c108acc6656882de0180e"},"cell_type":"markdown","source":"# Median length of the sentence in both the classes"},{"metadata":{"trusted":true,"_uuid":"e84805024e2c7e775ed7b54b2fc99ed989def7d8"},"cell_type":"code","source":"print (df_train[df_train['target']==1]['length'].median())\nprint (df_train[df_train['target']==0]['length'].median())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7e9e228f5f88d84c43a0201bb4a4ab84f01f3b2d"},"cell_type":"markdown","source":"# Functions for vocabulary building and checking coverage in different word embeddings"},{"metadata":{"trusted":true,"_uuid":"db68d83c2aacaaf3d0a9614975b43f923c80eb65"},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\n\ndef check_coverage(vocab, embeddings_index):\n    known_words = {}\n    unknown_words = {}\n    nb_known_words = 0\n    nb_unknown_words = 0\n    for word in vocab.keys():\n        try:\n            known_words[word] = embeddings_index[word]\n            nb_known_words += vocab[word]\n        except:\n            unknown_words[word] = vocab[word]\n            nb_unknown_words += vocab[word]\n            pass\n\n    print('Found embeddings for {:.3%} of vocab'.format(len(known_words) / len(vocab)))\n    print('Found embeddings for  {:.3%} of all text'.format(nb_known_words / (nb_known_words + nb_unknown_words)))\n    unknown_words = sorted(unknown_words.items(), key=operator.itemgetter(1))[::-1]\n    return unknown_words","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7a42ac8e15f116426a8bf243a46a6c88f3174043"},"cell_type":"markdown","source":"# function to add embeddings for lowercase words in the embeddings"},{"metadata":{"trusted":true,"_uuid":"03a34bc5a23089a5e523530a49a86b7ec303c5fa"},"cell_type":"code","source":"def add_lower(embedding, vocab):\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97935279b5a105858dc74902db849e91a07810de"},"cell_type":"markdown","source":"# lists to handle contractions, punctuations, mis spelled words"},{"metadata":{"trusted":true,"_uuid":"502f6365c393a9bbede31d47102f9afdc02a93cf"},"cell_type":"code","source":"contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\npunct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'\npunct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\nmispell_dict = {'examinaton': 'examination',\n                'undergraduation': 'under graduation',\n                'fiancé': 'fiance',\n                'qoura': 'quora',\n                'bhakts': 'followers',\n                'quorans': 'quora users',\n                'brexit': 'Britain exit',\n                'cryptocurrencies': 'cryptocurrency',\n                'colour': 'color',\n                'centre': 'center',\n                'favourite': 'favorite',\n                'travelling': 'traveling',\n                'counselling': 'counseling',\n                'theatre': 'theater',\n                'cancelled': 'canceled',\n                'labour': 'labor',\n                'organisation': 'organization',\n                'wwii': 'world war 2',\n                'citicise': 'criticize',\n                'youtu ': 'youtube ',\n                'Qoura': 'Quora',\n                'sallary': 'salary',\n                'Whta': 'What',\n                'narcisist': 'narcissist',\n                'howdo': 'how do',\n                'whatare': 'what are',\n                'howcan': 'how can',\n                'howmuch': 'how much',\n                'howmany': 'how many',\n                'whydo': 'why do',\n                'doI': 'do I',\n                'theBest': 'the best',\n                'howdoes': 'how does',\n                'mastrubation': 'masturbation',\n                'mastrubate': 'masturbate',\n                \"mastrubating\": 'masturbating',\n                'pennis': 'penis',\n                'Etherium': 'Ethereum',\n                'narcissit': 'narcissist',\n                'bigdata': 'big data',\n                '2k17': '2017',\n                '2k18': '2018',\n                'qouta': 'quota',\n                'exboyfriend': 'ex boyfriend',\n                'airhostess': 'air hostess',\n                \"whst\": 'what',\n                'watsapp': 'whatsapp',\n                'demonitisation': 'demonetization',\n                'demonitization': 'demonetization',\n                'demonetisation': 'demonetization',\n                'pokémon': 'pokemon',\n                'paralizing': 'paralising',\n                'perfeccionism': 'perfectionism',\n                'depreciaton': 'depreciation',\n                'abvicable': 'abdicable',\n                'catanation': 'catenation',\n                'leasership': 'leadership',\n                'webassembly': 'web assembly',\n                'fortitide': 'fortitude',\n                'withdrow': 'withdraw',\n                'bomblasts': 'bomb blasts',\n                'engineerer': 'engineer',\n                'citycarclean': 'city car clean',\n                'billionsites': 'billion sites',\n                'willhandjob': 'will hand job',\n                'fireguns': 'fire guns',\n                'justeat': 'just eat',\n                'ubereats': 'uber eats',\n                'doinformation': 'do information',\n                'freshersworld': 'freshers world',\n                'topicwise': 'topic wise',\n                'excitee': 'excited',\n                'bengalore': 'bangalore',\n                'proproetor': 'proprietor',\n                'migeration': 'migration',\n                'ejectulate': 'Ejaculate',\n                'glucoze': 'glucose',\n                'whatapp': 'whatsapp',\n                'sumup': 'sum up',\n                'besic': 'basic',\n                'experienceed': 'experienced',\n                'feminisam': 'feminism',\n                'kayboard': 'keyboard',\n                'retructuring': 'restructuring',\n                'becomd': 'become',\n                'preidct': 'predict',\n                'statups': 'startups',\n                'superbrains': 'super brains',\n                'becoome': 'become',\n                'gwroth': 'growth',\n                'wakeupnow': 'wake up now',\n                'headpone': 'headphone',\n                'industiry': 'industry',\n                'arichtecture': 'architecture',\n                'simlarity': 'similarity',\n                'walmartlabs': 'walmart labs',\n                'thunderstike': 'thunder stike',\n                'maintanable': 'maintainable',\n                'diffferently': 'differently',\n                'careamics': 'ceramics',\n                'sinnister': 'sinister',\n                'quoras': 'quora',\n                'breakimg': 'breaking',\n                'surggery': 'surgery',\n                'whatwill': 'what will',\n                'adhaar': 'identity',\n                'aidentity': 'identity',\n                'upwork': 'up work',\n                'alshamsi': 'al shamsi',\n                'litecoin': 'cryptocurrency ',\n                'chapterwise': 'chapter wise',\n                'blockchains': 'blockchain',\n                'flipcart': 'flipkart',\n               'Terroristan': 'terrorist Pakistan',\n                'terroristan': 'terrorist Pakistan',\n                'BIMARU': 'Bihar, Madhya Pradesh, Rajasthan, Uttar Pradesh',\n                'Hinduphobic': 'Hindu phobic',\n                'hinduphobic': 'Hindu phobic',\n                'Hinduphobia': 'Hindu phobic',\n                'hinduphobia': 'Hindu phobic',\n                'Babchenko': 'Arkady Arkadyevich Babchenko faked death',\n                'Boshniaks': 'Bosniaks',\n                'Dravidanadu': 'Dravida Nadu',\n                'mysoginists': 'misogynists',\n                'MGTOWS': 'Men Going Their Own Way',\n                'mongloid': 'Mongoloid',\n                'unsincere': 'insincere',\n                'meninism': 'male feminism',\n                'jewplicate': 'jewish replicate',\n                'unoin': 'Union',\n                'daesh': 'Islamic State of Iraq and the Levant',\n                'Kalergi': 'Coudenhove-Kalergi',\n                'Bhakts': 'Bhakt',\n                'bhakts': 'Bhakt',\n                'Tambrahms': 'Tamil Brahmin',\n                'Pahul': 'Amrit Sanskar',\n                'SJW': 'social justice warrior',\n                'SJWs': 'social justice warrior',\n                'incel': ' involuntary celibates',\n                'incels': ' involuntary celibates',\n                'emiratis': 'Emiratis',\n                'weatern': 'western',\n                'westernise': 'westernize',\n                'Pizzagate': 'Pizzagate conspiracy theory',\n                'naïve': 'naive',\n                'Skripal': 'Sergei Skripal',\n                'Remainers': 'British remainer',\n                'remainers': 'British remainer',\n                'bremainer': 'British remainer',\n                'antibrahmin': 'anti Brahminism',\n                'HYPSM': 'Harvard, Yale, Princeton, Stanford, MIT',\n                'HYPS': 'Harvard, Yale, Princeton, Stanford',\n                'kompromat': 'compromising material',\n                'Tharki': 'pervert',\n                'tharki': 'pervert',\n                'mastuburate': 'masturbate',\n                'Zoë': 'Zoe',\n                'indans': 'Indian',\n                'xender': 'gender',\n                'Naxali ': 'Naxalite ',\n                'Naxalities': 'Naxalites',\n                'Bathla': 'Namit Bathla',\n                'Mewani': 'Indian politician Jignesh Mevani',\n                'clichéd': 'cliche',\n                'cliché': 'cliche',\n                'clichés': 'cliche',\n                'Wjy': 'Why',\n                'Fadnavis': 'Indian politician Devendra Fadnavis',\n                'Awadesh': 'Indian engineer Awdhesh Singh',\n                'Awdhesh': 'Indian engineer Awdhesh Singh',\n                'Khalistanis': 'Sikh separatist movement',\n                'madheshi': 'Madheshi',\n                'BNBR': 'Be Nice, Be Respectful',\n                'Bolsonaro': 'Jair Bolsonaro',\n                'XXXTentacion': 'Tentacion',\n                'Padmavat': 'Indian Movie Padmaavat',\n                'Žižek': 'Slovenian philosopher Slavoj Žižek',\n                'Adityanath': 'Indian monk Yogi Adityanath',\n                'Brexit': 'British Exit',\n                'Brexiter': 'British Exit supporter',\n                'Brexiters': 'British Exit supporters',\n                'Brexiteer': 'British Exit supporter',\n                'Brexiteers': 'British Exit supporters',\n                'Brexiting': 'British Exit',\n                'Brexitosis': 'British Exit disorder',\n                'brexit': 'British Exit',\n                'brexiters': 'British Exit supporters',\n                'jallikattu': 'Jallikattu',\n                'fortnite': 'Fortnite ',\n                'Swachh': 'Swachh Bharat mission campaign ',\n                'Quorans': 'Quoran',\n                'Qoura ': 'Quora ',\n                'quoras': 'Quora',\n                'Quroa': 'Quora',\n                'QUORA': 'Quora',\n                'narcissit': 'narcissist',\n                # extra in sample\n                'Doklam': 'Tibet',\n                'Drumpf': 'Donald Trump fool',\n                'Drumpfs': 'Donald Trump fools',\n                'Strzok': 'Hillary Clinton scandal',\n                'rohingya': 'Rohingya ',\n                'wumao': 'cheap Chinese stuff',\n                'wumaos': 'cheap Chinese stuff',\n                'Sanghis': 'Sanghi',\n                'Tamilans': 'Tamils',\n                'biharis': 'Biharis',\n                'Rejuvalex': 'hair growth formula',\n                'Feku': 'Fake',\n                'deplorables': 'deplorable',\n                'muhajirs': 'Muslim immigrant',\n                'Gujratis': 'Gujarati',\n                'Chutiya': 'Fucker',\n                'Chutiyas': 'Fucker',\n                'thighing': 'masturbate',\n                '卐': 'Nazi Germany',\n                'Pribumi': 'Native Indonesian',\n                'Gurmehar': 'Gurmehar Kaur Indian student activist',\n                'Novichok': 'Soviet Union agents',\n                'Khazari': 'Khazars',\n                'Demonetization': 'demonetization',\n                'demonetisation': 'demonetization',\n                'demonitisation': 'demonetization',\n                'demonitization': 'demonetization',\n                'demonetisation': 'demonetization',\n                'cryptocurrencies': 'cryptocurrency',\n                'Hindians': 'North Indian who hate British',\n                'vaxxer': 'vocal nationalist ',\n                'remoaner': 'remainer ',\n                'bremoaner': 'British remainer ',\n                'Jewism': 'Judaism',\n                'Eroupian': 'European',\n                'WMAF': 'White male married Asian female',\n                'moeslim': 'Muslim',\n                'cishet': 'cisgender and heterosexual person',\n                'Eurocentric': 'Eurocentrism ',\n                'Jewdar': 'Jew dar',\n                'Asifa': 'abduction, rape, murder case ',\n                'marathis': 'Marathi',\n                'Trumpanzees': 'Trump chimpanzee fool',\n                'Crimean': 'Crimea people ',\n                'atrracted': 'attract',\n                'LGBT': 'lesbian, gay, bisexual, transgender',\n                'Boshniak': 'Bosniaks ',\n                'Myeshia': 'widow of Green Beret killed in Niger',\n                'demcoratic': 'Democratic',\n                'raaping': 'rape',\n                'Dönmeh': 'Islam',\n                'feminazism': 'feminism nazi',\n                'langague': 'language',\n                'Hongkongese': 'HongKong people',\n                'hongkongese': 'HongKong people',\n                'Kashmirians': 'Kashmirian',\n                'Chodu': 'fucker',\n                'penish': 'penis',\n                'micropenis': 'tiny penis',\n                'Madridiots': 'Real Madrid idiot supporters',\n                'Ambedkarite': 'Dalit Buddhist movement ',\n                'ReleaseTheMemo': 'cry for the right and Trump supporters',\n                'harrase': 'harass',\n                'Barracoon': 'Black slave',\n                'Castrater': 'castration',\n                'castrater': 'castration',\n                'Rapistan': 'Pakistan rapist',\n                'rapistan': 'Pakistan rapist',\n                'Turkified': 'Turkification',\n                'turkified': 'Turkification',\n                'Dumbassistan': 'dumb ass Pakistan',\n                'facetards': 'Facebook retards',\n                'rapefugees': 'rapist refugee',\n                'superficious': 'superficial',\n                # extra from kagglers\n                'colour': 'color',\n                'centre': 'center',\n                'favourite': 'favorite',\n                'travelling': 'traveling',\n                'counselling': 'counseling',\n                'theatre': 'theater',\n                'cancelled': 'canceled',\n                'labour': 'labor',\n                'organisation': 'organization',\n                'wwii': 'world war 2',\n                'citicise': 'criticize',\n                'youtu ': 'youtube ',\n                'sallary': 'salary',\n                'Whta': 'What',\n                'narcisist': 'narcissist',\n                'narcissit': 'narcissist',\n                'howdo': 'how do',\n                'whatare': 'what are',\n                'howcan': 'how can',\n                'howmuch': 'how much',\n                'howmany': 'how many',\n                'whydo': 'why do',\n                'doI': 'do I',\n                'theBest': 'the best',\n                'howdoes': 'how does',\n                'mastrubation': 'masturbation',\n                'mastrubate': 'masturbate',\n                'mastrubating': 'masturbating',\n                'pennis': 'penis',\n                'Etherium': 'Ethereum',\n                'bigdata': 'big data',\n                '2k17': '2017',\n                '2k18': '2018',\n                'qouta': 'quota',\n                'exboyfriend': 'ex boyfriend',\n                'airhostess': 'air hostess',\n                'whst': 'what',\n                'watsapp': 'whatsapp',\n                # extra\n                'bodyshame': 'body shaming',\n                'bodyshoppers': 'body shopping',\n                'bodycams': 'body cams',\n                'Cananybody': 'Can any body',\n                'deadbody': 'dead body',\n                'deaddict': 'de addict',\n                'Northindian': 'North Indian ',\n                'northindian': 'north Indian ',\n                'northkorea': 'North Korea',\n                'Whykorean': 'Why Korean',\n                'koreaboo': 'Korea boo ',\n                'Brexshit': 'British Exit bullshit',\n                'shithole': 'shithole ',\n                'shitpost': 'shit post',\n                'shitslam': 'shit Islam',\n                'shitlords': 'shit lords',\n                'Fck': 'Fuck',\n                'fck': 'fuck',\n                'Clickbait': 'click bait ',\n                'clickbait': 'click bait ',\n                'mailbait': 'mail bait',\n                'healhtcare': 'healthcare',\n                'trollbots': 'troll bots',\n                'trollled': 'trolled',\n                'trollimg': 'trolling',\n                'cybertrolling': 'cyber trolling',\n                'sickular': 'India sick secular ',\n                'suckimg': 'sucking',\n                'Idiotism': 'idiotism',\n                'Niggerism': 'Nigger',\n                'Niggeriah': 'Nigger'}\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"62539796c9297b910e12b1a5a0b728cd7c3376e3"},"cell_type":"markdown","source":"# Different Embeddings"},{"metadata":{"trusted":true,"_uuid":"6b79605e764df16f996937249e47f573ae5ef7af"},"cell_type":"code","source":"glove = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\nparagram =  '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\nwiki_news = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n\n\n\ndef load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    \n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n        \n    return embeddings_index\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eaec69ff262ed73c802e0104e564359029a07cc1"},"cell_type":"markdown","source":"# Choose Embedding"},{"metadata":{"trusted":true,"_uuid":"83a61cc58829e18c65ac9b61b06c24c1b57a7029"},"cell_type":"markdown","source":"choose from glove or paragram or wiki_news\nembed_pretrained = load_embed(glove)\nprint (len(embed_pretrained))"},{"metadata":{"_uuid":"37f6b1dfb16dbe9f3dae9bac992900619aee4b9d"},"cell_type":"markdown","source":"wiki_embeddings = load_embed(wiki_news)\nprint (len(wiki_embeddings))"},{"metadata":{"trusted":true,"_uuid":"ec9f985c04c3e6f2755dd2a6e6be3e762c566ca0"},"cell_type":"code","source":"glove_embeddings = load_embed(glove)\nprint (len(glove_embeddings))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8d7b405e097841c735a143addb1a772869d3a233"},"cell_type":"markdown","source":"paragram_embeddings = load_embed(paragram)\nprint (len(paragram_embeddings))"},{"metadata":{"_uuid":"dd7128ec17831b6339099fcc6a660992491c7d75"},"cell_type":"markdown","source":"# Build Vocabulary"},{"metadata":{"trusted":true,"_uuid":"6b19b7e1f4d0734603b71b226f94767ad7c8d677"},"cell_type":"code","source":"train = df_train['question_text']\ntest = df_test['question_text']\ndf = pd.concat([train ,test])\n\nvocab = build_vocab(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e0b01408654552ec67d590c8250af30ad4aa3da"},"cell_type":"markdown","source":"print(\"oov : paragram \")\noov = check_coverage(vocab, paragram_embeddings)\n\nadd_lower(paragram_embeddings, vocab)\n\nprint(\"oov : \")\noov = check_coverage(vocab, paragram_embeddings)"},{"metadata":{"_uuid":"87dd218f9e0a56a16299a474f9de04b68201973e"},"cell_type":"markdown","source":"# Check Coverage in Glove embeddings before and after adding lower words"},{"metadata":{"trusted":true,"_uuid":"692f68c96fdd44a65551032fda1bc082c083fbff"},"cell_type":"code","source":"print(\"oov : Glove \")\noov = check_coverage(vocab, glove_embeddings)\n\nadd_lower(glove_embeddings, vocab)\n\nprint(\"oov : \")\noov = check_coverage(vocab, glove_embeddings)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9091ef7d133e6f7c71cdc2891a492b24577ecbcb"},"cell_type":"markdown","source":"# functions to handle contractions"},{"metadata":{"trusted":true,"_uuid":"c8ca7f0c849366b1a3f377245aea08ed24ebf979"},"cell_type":"code","source":"def known_contractions(embed):\n    known = []\n    for contract in contraction_mapping:\n        if contract in embed:\n            known.append(contract)\n    return known\n\ndef clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"03d7707a1ff206cb03752d5a8a721f939bf2f9e8"},"cell_type":"markdown","source":"# functions to handle special characters"},{"metadata":{"trusted":true,"_uuid":"5003c91cf05e030d8c69262e7e208e6560f707b0"},"cell_type":"code","source":"def clean_special_chars(text, punct, puncts, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    for p in puncts:\n        text = text.replace(p, f' {p} ')\n        \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fac44d1b82754b7ed0dabe538f38c4a56e11ce12"},"cell_type":"markdown","source":"# function to handle wrong spellings"},{"metadata":{"trusted":true,"_uuid":"f6f7fd086f0bdb5f77f16a024a9e60d630552902"},"cell_type":"code","source":"def correct_spelling(x, dic):\n    for word in dic.keys():\n        x = x.replace(word, dic[word])\n    return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7036be1d12f774a359732e234e066db5a07cd096"},"cell_type":"markdown","source":"# function to handle numerical characters"},{"metadata":{"trusted":true,"_uuid":"ee91e82511e1d3dd9ef009fd705edd25270070ea"},"cell_type":"code","source":"import re\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42024a7e4dc7a155c2edb88a01e8e95c11731e93"},"cell_type":"markdown","source":"from nltk.stem import PorterStemmer\nfrom textblob import Word\nstemmer = PorterStemmer()"},{"metadata":{"_uuid":"b6edbc283bdb17d15c8d273bd804c0095d930887"},"cell_type":"markdown","source":"# Data processings on DF train"},{"metadata":{"trusted":true,"_uuid":"7a73ce00d8f597ed705e022914f7536e1a62fca1"},"cell_type":"code","source":"# Lowering\ndf_train['question_text'] = df_train['question_text'].apply(lambda x: x.lower())\n# Contractions\ndf_train['question_text'] = df_train['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))\n# Special characters\ndf_train['question_text'] = df_train['question_text'].apply(lambda x: clean_special_chars(x, punct, puncts, punct_mapping))\n# Spelling mistakes\ndf_train['question_text'] = df_train['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))\n# Clean Numbers\ndf_train['question_text'] = df_train['question_text'].apply(lambda x: clean_numbers(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"852a1e61d149fef1511d5ecd4e508baefcb90883"},"cell_type":"markdown","source":"# Data processings on DF test"},{"metadata":{"trusted":true,"_uuid":"6fe4082f425a8d0e88e35170e534163d2ee9523b"},"cell_type":"code","source":"# Lowering\ndf_test['question_text'] = df_test['question_text'].apply(lambda x: x.lower())\n# Contractions\ndf_test['question_text'] = df_test['question_text'].apply(lambda x: clean_contractions(x, contraction_mapping))\n# Special characters\ndf_test['question_text'] = df_test['question_text'].apply(lambda x: clean_special_chars(x, punct,puncts, punct_mapping))\n# Spelling mistakes\ndf_test['question_text'] = df_test['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))\n# clean numbers\ndf_test['question_text'] = df_test['question_text'].apply(lambda x: clean_numbers(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bf909f0c459587503494e20fafaf5333f71b796b"},"cell_type":"markdown","source":"# Build vocab again and check coverage after the above data handling"},{"metadata":{"trusted":true,"_uuid":"c54d10e544837ce2f191afb0af3abbbce22dc74e"},"cell_type":"code","source":"train = df_train['question_text']\ntest = df_test['question_text']\ndf = pd.concat([train ,test])\n\nvocab = build_vocab(df)\n\nprint(\"oov : \")\noov = check_coverage(vocab, glove_embeddings)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91dd06f7e6fdf7d7c12cf346f420bd7da048ca6e"},"cell_type":"markdown","source":"# Check OOV (out of vocabulary)"},{"metadata":{"trusted":true,"_uuid":"c429e1b6d499ad3f4c64b72686f41484a6216dee"},"cell_type":"code","source":"#oov","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5ee34cd502257bdb35a551a1d3128dbb160bec32"},"cell_type":"markdown","source":"# Split the data into train, holdout"},{"metadata":{"trusted":true,"_uuid":"ed8731cd233858da78ff7624834153129310c884"},"cell_type":"code","source":"#train_Y=df_train['target']\n#train_X=df_train['question_text']\nfrom sklearn.model_selection import train_test_split\n#X_train, X_test2, y_train, y_test2 = train_test_split(train_X, train_Y, test_size=0.04, random_state=123)\n#X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.045, random_state=123)\n\ntrain_df, test2_df = train_test_split(df_train, test_size=0.04, random_state=123)\ntrain_df, valid_df = train_test_split(train_df, test_size=0.00001, random_state=123)\n\ntrain_df=train_df.reset_index(drop=True)\nvalid_df=valid_df.reset_index(drop=True)\ntest2_df=test2_df.reset_index(drop=True)\n\nX_train=train_df['question_text'].values\ny_train=train_df['target'].values\n\n\nX_test2=test2_df['question_text'].values\ny_test2=test2_df['target'].values\n\n\nX_valid=valid_df['question_text'].values\ny_valid=valid_df['target'].values\n\n\nX_test=df_test['question_text'].values\n\n\n\nprint (X_train.shape)\nprint (y_train.shape)\n\n\nprint (X_valid.shape)\nprint (y_valid.shape)\n\n\n\nprint (X_test2.shape)\nprint (y_test2.shape)\n\n\nprint (X_test.shape)\n\n\nprint (train_df.shape)\nprint (valid_df.shape)\nprint (test2_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ca54d7f5a4c733a57d5b9193dd3b68c7ce62e54"},"cell_type":"code","source":"X_len_train=train_df['length'].values\nX_len_test2=test2_df['length'].values\nX_len_valid=valid_df['length'].values\nX_len_test=df_test['length'].values\n\nprint (X_len_train.shape)\nprint (X_len_test2.shape)\nprint (X_len_valid.shape)\nprint (X_len_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c86a7cbf32c8389961b72b17807599275dc8962e"},"cell_type":"markdown","source":"# Get tokens for words using keras tokenizer"},{"metadata":{"trusted":true,"_uuid":"8219858d80cbae384a5d248b387d271659373805"},"cell_type":"code","source":"embedding_dim = 300 # how big is each word vector\nmax_features = 120000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70 # max number of words in a question to use\n\nfrom keras.preprocessing.text import Tokenizer\ntokenizer = Tokenizer(num_words=max_features)\n#tokenizer = Tokenizer()\ntokenizer.fit_on_texts(X_train.tolist() + X_valid.tolist()+ X_test2.tolist() + X_test.tolist())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b9d5417eae81f271b6f0dce812d28544c3025c5"},"cell_type":"markdown","source":"# Vocab Size"},{"metadata":{"trusted":true,"_uuid":"feafd8b9db4ae4a1f234df8c518f9e004032c068"},"cell_type":"code","source":"vocab_size = len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index\nprint (vocab_size)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"782ead15cd69d8f578637537db5099ecfefab742"},"cell_type":"code","source":"#tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9829fd7f567ed71f589567dfe23633a3ca8ee67"},"cell_type":"markdown","source":"# Tokenize the sentences : convert sentences to sequence of tokens (indices or numbers)"},{"metadata":{"trusted":true,"_uuid":"60718fda57132e185ff49f844248892937ea0fc9"},"cell_type":"markdown","source":"x_train = tokenizer.texts_to_sequences(X_train)\nx_valid = tokenizer.texts_to_sequences(X_valid)\nx_test2 = tokenizer.texts_to_sequences(X_test2)\nx_test = tokenizer.texts_to_sequences(X_test)"},{"metadata":{"_uuid":"a8e91d9b1d58e976552329a48718438450ad5505"},"cell_type":"markdown","source":"# Pad the sequences with 0s so that all sentences/sequences have same length for NN"},{"metadata":{"trusted":true,"_uuid":"93c295098054bcf6f7452f550fbdfe3763e186e5"},"cell_type":"code","source":"#maxlen=70","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c56288207593d8734e92ddea17c0ee3df85bec97"},"cell_type":"markdown","source":"from keras.preprocessing.sequence import pad_sequences\n\nx_train = pad_sequences(x_train, padding='post', maxlen=maxlen)\nx_valid = pad_sequences(x_valid, padding='post', maxlen=maxlen)\nx_test2 = pad_sequences(x_test2, padding='post', maxlen=maxlen)\nx_test = pad_sequences(x_test, padding='post', maxlen=maxlen)"},{"metadata":{"trusted":true,"_uuid":"4292b2e210d89c1c01b0cd970e0e673150a9b336"},"cell_type":"markdown","source":"print(x_train[0])\nprint(X_train[0])\nprint(x_test[0])\nprint(X_test[0])"},{"metadata":{"_uuid":"d0f321e6f6ed1d2f7238ba201a8ab4667213ec97"},"cell_type":"markdown","source":"# function to extract embedding vectors for each word"},{"metadata":{"trusted":true,"_uuid":"67a7d05abb24d20b7553f6432db8643d68b28f6c"},"cell_type":"code","source":"def index_to_matrix(embeddings_index,word_index):\n    embedding_matrix = np.zeros((len(word_index) + 1, embedding_dim))\n    for word, i in word_index.items():\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None:\n            # words not found in embedding index will be all-zeros.\n            embedding_matrix[i] = embedding_vector\n    return (embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"668866bb17f8ff6d5bbe31d1c0adacacb7274ab4"},"cell_type":"markdown","source":"# Getting glove embeddings"},{"metadata":{"trusted":true,"_uuid":"56f7d8133ec9294ade25b71cd28ee68f8b1227ec"},"cell_type":"code","source":"glove_embedding_matrix=index_to_matrix(glove_embeddings,tokenizer.word_index)\nembedding_matrix=glove_embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f1cac1f3501e4b0273735aec339472ee54562627"},"cell_type":"markdown","source":"paragram_embedding_matrix=index_to_matrix(paragram_embeddings,tokenizer.word_index)\nembedding_matrix=paragram_embedding_matrix"},{"metadata":{"trusted":true,"_uuid":"138b3809ef3d316b2dfb87d4f19493a691930c09"},"cell_type":"markdown","source":"import gc\ngc.collect()\ndel glove_embeddings\ngc.collect()"},{"metadata":{"_uuid":"06e766cd3adb3e7451d70fe7e03280742e7ac0d3"},"cell_type":"markdown","source":"# Import Libraries"},{"metadata":{"trusted":true,"_uuid":"8d9cc018e331effdd720b71fe9df89141919873f"},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Dense, Embedding, Bidirectional, CuDNNGRU,CuDNNLSTM, GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, Input, Dropout, Add\nfrom keras.optimizers import Adam\nfrom keras.models import Sequential\nfrom keras import layers\nimport keras.callbacks\nfrom keras.optimizers import Adam\nfrom keras.callbacks import ModelCheckpoint","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eeed512811be7299cfef2b244a4c451948995b8a"},"cell_type":"markdown","source":"# Define Model"},{"metadata":{"trusted":true,"_uuid":"30b2d2f7f3033b7d697c1714b06abe2fb92f3ddf"},"cell_type":"code","source":"def make_model(embedding_matrix, maxlen, embed_size=300, loss='binary_crossentropy'):\n    inp    = Input(shape=(maxlen,))\n    inp2   = Input(shape=(1,))\n    x      = Embedding(vocab_size, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    #x      = Bidirectional(CuDNNGRU(256, return_sequences=True))(x)\n    #x      = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)\n    x      = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x      = Dropout(0.2)(x)\n    x      = Bidirectional(CuDNNLSTM(64, return_sequences=True))(x)\n    #x      = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)\n    #x      = Attention(maxlen)(x)\n    #x      = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)\n    x      = Dropout(0.2)(x)\n    #x      = Bidirectional(CuDNNLSTM(32, return_sequences=True))(x)\n    #x      = Dropout(0.25)(x)\n    avg_pl = GlobalAveragePooling1D()(x)\n    max_pl = GlobalMaxPooling1D()(x)\n    concat = concatenate([avg_pl, max_pl])\n    #add=Add()([concat, inp2])\n    #concat = concatenate([avg_pl, max_pl,inp2])      # using sentence length as one of the feature.\n    dense1  = Dense(32, activation=\"relu\")(concat)\n    #dense1  = Dense(32, activation=\"relu\")(concat)\n    concat = concatenate([dense1, inp2])\n    #drop1   = Dropout(0.2)(concat)\n    #dense2  = Dense(8, activation=\"relu\")(drop1)\n    #drop2   = Dropout(0.1)(dense2)\n    #dense3  = Dense(8, activation=\"relu\")(dense1)\n    output = Dense(1, activation=\"sigmoid\")(concat)\n    \n    #model  = Model(inputs=inp, outputs=output)\n    model = Model(inputs=[inp,inp2], outputs=output)\n    model.compile(optimizer='adam',loss='binary_crossentropy',metrics=['accuracy'])\n    #model.compile(optimizer=Adam(lr=0.0001),loss='binary_crossentropy',metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9bc3237d5801eade1e8367876cce62a671d8cb76"},"cell_type":"markdown","source":"# Make model instance"},{"metadata":{"trusted":true,"_uuid":"5da72d953d6f40f16bdd826c15dbf9df1101c802"},"cell_type":"code","source":"model = make_model(embedding_matrix,maxlen=70)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca2bf1ca517ac2ffcb6f35ef61abb438cd4cf0e9"},"cell_type":"markdown","source":"# Model summary"},{"metadata":{"trusted":true,"_uuid":"c5f5500371064301946d97e5429391a16949eb72"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1411f13f9e9d1f58859c63c92d8233e9c67bdbcd"},"cell_type":"markdown","source":"checkpointer = ModelCheckpoint(filepath='model.weights.best.hdf5',monitor='val_loss', mode='auto', verbose = 1, save_best_only=True)\n"},{"metadata":{"_uuid":"dc0d8bda471ed133501582aa87228d569cc4ec61"},"cell_type":"markdown","source":"# function to plot accuracy and loss"},{"metadata":{"trusted":true,"_uuid":"3ec08a99ffed551ddec2a35a7cfda064e1631c17"},"cell_type":"code","source":"import matplotlib.pyplot as plt \nplt.style.use('ggplot')\n\ndef plot_history(history):\n    loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' not in s]\n    val_loss_list = [s for s in history.history.keys() if 'loss' in s and 'val' in s]\n    acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' not in s]\n    val_acc_list = [s for s in history.history.keys() if 'acc' in s and 'val' in s]\n    \n    if len(loss_list) == 0:\n        print('Loss is missing in history')\n        return \n    \n    ## As loss always exists\n    epochs = range(1,len(history.history[loss_list[0]]) + 1)\n    \n    ## Loss\n    plt.figure(1)\n    for l in loss_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n    for l in val_loss_list:\n        plt.plot(epochs, history.history[l], 'g', label='Validation loss (' + str(str(format(history.history[l][-1],'.5f'))+')'))\n    \n    plt.title('Loss')\n    plt.xlabel('Epochs')\n    plt.ylabel('Loss')\n    plt.legend()\n    \n    ## Accuracy\n    plt.figure(2)\n    for l in acc_list:\n        plt.plot(epochs, history.history[l], 'b', label='Training accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n    for l in val_acc_list:    \n        plt.plot(epochs, history.history[l], 'g', label='Validation accuracy (' + str(format(history.history[l][-1],'.5f'))+')')\n\n    plt.title('Accuracy')\n    plt.xlabel('Epochs')\n    plt.ylabel('Accuracy')\n    plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34aae10e6e7b9b8a56795e281d42c14899b9f721"},"cell_type":"markdown","source":"# k fold splitting "},{"metadata":{"trusted":true,"_uuid":"70f9d805729498f4d54b27e4881d550062d12a46"},"cell_type":"code","source":"from sklearn.model_selection import StratifiedKFold\nseed = 7\nn_splits=5\nnp.random.seed(seed)\nkfold = StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=seed)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4853f30719a5a415acccb059fb90ccd346be0a07"},"cell_type":"markdown","source":"# Running model for kfold"},{"metadata":{"trusted":true,"_uuid":"2cb9005b6d97457ec37ec5adc9cfa112a7c6dbf4","scrolled":false},"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\ni=1\nmaxlength={}\nmaxl={}\nfor train, valid in kfold.split(X_train, y_train):\n    #print ((train))\n    if i <=n_splits:\n        maxl[i]=X_len_train[train].max()\n        print (X_len_train[train])\n        maxlength[i]=int(np.quantile(X_len_train[train],0.999))\n        print (\"Running Fold\", i, \"/\", n_splits)\n        print (\"split 99.9 percentile length\",maxlength[i],\"split max length\",maxl[i])\n        modelname=str(\"Model\") + str(i)\n        #print (train)\n        #print (valid)\n        train_data=[X_train[j] for j in train]\n        valid_data=[X_train[k] for k in valid]\n        \n        #print (train_data[0])\n        #print (valid_data[0])\n        \n        train_data = tokenizer.texts_to_sequences(train_data)\n        valid_data = tokenizer.texts_to_sequences(valid_data)\n        train_data = pad_sequences(train_data, padding='post', maxlen=maxlength[i])\n        valid_data = pad_sequences(valid_data, padding='post', maxlen=maxlength[i])\n        #print (train_data[0])\n        #print (valid_data[0])\n        #x_test2 = pad_sequences(x_test2, padding='post', maxlen=maxlen)\n        #x_test = pad_sequences(x_test, padding='post', maxlen=maxlen)\n        #print (x_train[train][:10])\n        #print ('before')\n        model = make_model(embedding_matrix,maxlength[i])\n        \n        #print ('after')        \n        #print (\"train\", x_train[train][:1])\n        #print (\"train\", y_train[train][:100])\n        print (\"train\", y_train[train].sum())\n        print (\"validation\",y_train[valid].sum())\n        checkpointer = ModelCheckpoint(filepath=modelname,monitor='val_loss', mode='auto', verbose = 1, save_best_only=True)\n        history = model.fit([train_data,X_len_train[train]], y_train[train],\n                        epochs=5,\n                        validation_data=([valid_data,X_len_train[valid]], y_train[valid]),\n                        batch_size=512,callbacks=[checkpointer])\n        plot_history(history)\n        i=i+1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1c13620d41ef533bcb6d46d24a1d13a38b64e80d"},"cell_type":"code","source":"#plot_history(history)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d2c39a8a3219ea1a849b159c34f5f2ea474f52f8"},"cell_type":"markdown","source":"# Make prediction on holdout, find threshhold cutoff which gives maximum F1 score on hold out, for each model"},{"metadata":{"trusted":true,"_uuid":"29f25b2c55b57ac6a5d4f21125b56eef36c8a935"},"cell_type":"code","source":"y_pred={}\ny_pred_test={}\n\ncount=0\nfor i in np.arange(1, n_splits+1, 1):\n    try:\n        model.load_weights(str(\"Model\") + str(i))\n        print (str(\"Model\") + str(i))\n        import sklearn\n        from sklearn.metrics import f1_score\n        #y_pred[i] = model.predict(x_test2, batch_size=512, verbose=1)\n        #y_pred_test[i] = model.predict(x_test, batch_size=512, verbose=1)\n        \n        x_test2 = tokenizer.texts_to_sequences(X_test2)\n        x_test = tokenizer.texts_to_sequences(X_test)\n        x_test2 = pad_sequences(x_test2, padding='post', maxlen=maxlength[i])\n        x_test = pad_sequences(x_test, padding='post', maxlen=maxlength[i])\n        \n        y_pred[i] = model.predict([x_test2,X_len_test2], batch_size=512, verbose=1)\n        y_pred_test[i] = model.predict([x_test,X_len_test], batch_size=512, verbose=1)\n       \n        \n        model_f1_score={}\n        for thresh in np.arange(0.1, 0.91, 0.01):\n            thresh = np.round(thresh, 2)\n            #print(\"F1 score at threshold {0} is {1}\".format(thresh, sklearn.metrics.f1_score(y_valid, (y_pred>=thresh).astype(int))))\n            model_f1_score[thresh]=sklearn.metrics.f1_score(y_test2, (y_pred[i]>=thresh).astype(int))\n               \n        model_cutoff=max(model_f1_score, key=model_f1_score.get)\n        print(\"Max F1 score  is {1} found at threshold {0}\".format(model_cutoff, model_f1_score[model_cutoff]))\n        count=count+1\n        #y_pred_final=y_pred_final + y_pred[i]\n    except:\n        pass\n\nprint ('count is ', count)\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c80a205c68cbe2d73a53a8fb84cc52f5676711a4"},"cell_type":"markdown","source":"# take average of predictions from all models"},{"metadata":{"trusted":true,"_uuid":"c844bece5311f9bfc3c7e1dd88544a240e51b0b6"},"cell_type":"code","source":"y_pred_final={}\ny_pred_test_final={}\nfor i in np.arange(1, count+1, 1):\n    if (i == 1):\n        y_pred_final=y_pred[i]\n        y_pred_test_final=y_pred_test[i]\n        #print (i)\n    else:\n        y_pred_final=y_pred_final + y_pred[i]\n        y_pred_test_final=y_pred_test_final + y_pred_test[i] \n        #print(i)\n    \n#print (count)\ny_pred_final=y_pred_final/count\ny_pred_test_final=y_pred_test_final/count\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"804686957872852899516d847be36e409253f438"},"cell_type":"markdown","source":"# find threshold cutoff on holdout for final model"},{"metadata":{"trusted":true,"_uuid":"edb049481ab1bbf76210b80b26f0e6b3aaff882c"},"cell_type":"code","source":"print (\"Final Model on hold out\") \nmodel_f1_score={}\nfor thresh in np.arange(0.1, 0.91, 0.01):\n    thresh = np.round(thresh, 2)\n    #print(\"F1 score at threshold {0} is {1}\".format(thresh, sklearn.metrics.f1_score(y_valid, (y_pred>=thresh).astype(int))))\n    model_f1_score[thresh]=sklearn.metrics.f1_score(y_test2, (y_pred_final>=thresh).astype(int))\n\nmodel_cutoff=max(model_f1_score, key=model_f1_score.get)\nprint(\"Max F1 score  is {1} found at threshold {0}\".format(model_cutoff, model_f1_score[model_cutoff]))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"827cb6474f3415f2b9f8328c2a70975a4e3c0c96"},"cell_type":"markdown","source":"# find accuracy, precision, recall , auc , f1 on holdout"},{"metadata":{"trusted":true,"_uuid":"a6c99a020692d956b0344bcb60df8cd01ae218c3"},"cell_type":"code","source":"y_pred_test2_final_class = (y_pred_final >= model_cutoff).astype(int) \nprint (y_test2.sum())\nprint (y_pred_test2_final_class.sum())\n\nimport sklearn\nfrom sklearn.metrics import f1_score\nprint(sklearn.metrics.f1_score(y_test2, y_pred_test2_final_class)) \n\nprint ('classification report')\nfrom sklearn.metrics import classification_report\nprint(classification_report(y_test2, y_pred_test2_final_class))\n\nprint ('Confusion matrix')\nprint(sklearn.metrics.confusion_matrix(y_test2, y_pred_test2_final_class)) \n\nfrom sklearn.metrics import roc_auc_score\nprint ('roc score')\nprint(sklearn.metrics.roc_auc_score(y_test2, y_pred_test2_final_class))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"63da353f2042f1491885cfa024b2f815b6db4c6d"},"cell_type":"markdown","source":"# Create df with actual and predicted values of holdout"},{"metadata":{"_uuid":"e56fb391c88c902fe6fe183c367594d28701ad44","trusted":true},"cell_type":"code","source":"df=pd.DataFrame(columns=['text','y_actual', 'y_pred','y_pred_prob','length'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ac2a046232f790ac45c036eb471de9f94829d6e7"},"cell_type":"code","source":"df['text'] = X_test2\ndf['y_pred'] =y_pred_test2_final_class\ndf['y_pred_prob'] =y_pred_final\ndf['length'] =X_len_test2\ndf['y_actual'] =y_test2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"eee715b781fe57c9160b26e010a58c503ce5e9bf"},"cell_type":"markdown","source":"# check records which are wrongly predicted and fall close to threshold"},{"metadata":{"trusted":true,"_uuid":"c881c75b01059c339782b7a41b7879160bb8008c"},"cell_type":"code","source":"df[(df.y_actual != df.y_pred ) & (df.y_pred_prob >=(model_cutoff-0.3)) & (df.y_pred_prob <=(model_cutoff + 0.3))]['text'].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9638c925777f81f5ff9f54b72e8c98895717f0a"},"cell_type":"markdown","source":"# False Negative records"},{"metadata":{"trusted":true,"_uuid":"5297d409b3a036988bd1f866ab6ef4dba5106174"},"cell_type":"code","source":"FN_phrases = df[(df.y_actual == 1) & (df.y_pred == 0)]\nFN_phrases.shape\n\nFN_words = []\nfor t in FN_phrases.text:\n    FN_words.append(t)\nFN_words[:10]\n\nFN_text = pd.Series(FN_words).str.cat(sep=' ')\nFN_text[:100]\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"90c862848d9bc4b58bc0b164f90b0594c59d5355"},"cell_type":"markdown","source":"# False negative wordcloud"},{"metadata":{"trusted":true,"_uuid":"0c0d6cfa0a9a3e4ce86dfc5fade1298ce72cdc66"},"cell_type":"code","source":"from wordcloud import WordCloud\nwordcloud = WordCloud(width=1600, height=800, max_font_size=200).generate(FN_text)\nplt.figure(figsize=(12,10))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e04b689101d6ad649128657498ce6e72a9394429"},"cell_type":"code","source":"from collections import Counter\ncounts = Counter(FN_text.split())\nprint(counts)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2488bca63947f82507750663526673e5509cc276"},"cell_type":"markdown","source":"# False Positive wordcloud"},{"metadata":{"trusted":true,"_uuid":"c79b6529eb48e90375dc29932d8a8bf8b57fa751"},"cell_type":"code","source":"FP_phrases = df[(df.y_actual == 0) & (df.y_pred == 1)]\nFP_phrases.shape\n\n\nFP_words = []\nfor t in FP_phrases.text:\n    FP_words.append(t)\n#print (FP_words[:10])\n\nFP_text = pd.Series(FP_words).str.cat(sep=' ')\n#print (FP_text[:100])\n\nfrom wordcloud import WordCloud\nwordcloud = WordCloud(width=1600, height=800, max_font_size=200).generate(FP_text)\nplt.figure(figsize=(12,10))\nplt.imshow(wordcloud, interpolation='bilinear')\nplt.axis(\"off\")\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aa4791f5a9391b0bf0738df0a35fac23c94172df"},"cell_type":"code","source":"from collections import Counter\ncounts = Counter(FP_text.split())\nprint(counts)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b5197421346944a89a7a6196c86e34479d636ef"},"cell_type":"markdown","source":"# On final test data"},{"metadata":{"trusted":true,"_uuid":"8d44d0fc6288e232e40dd11f8bfb77b1ee70a40d"},"cell_type":"code","source":"df_test['prediction']= (y_pred_test_final >= model_cutoff).astype(int) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4dad7e533afa4e0ba94d3f392db2317a6d54264e"},"cell_type":"code","source":"#df_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"398f9db258281e6e18dd778d383f522ac9c6357e"},"cell_type":"code","source":"df_test=df_test.drop(['question_text'], axis=1)\ndf_test=df_test.drop(['length'], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"54c6bb8098723d37e79f2edecfcfec418251fd0a"},"cell_type":"markdown","source":"# Write to csv"},{"metadata":{"trusted":true,"_uuid":"694cb9ac5d2b1c474c14b2168547a9dbbf94774e"},"cell_type":"code","source":"df_test.to_csv(r'submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}