{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport re\nimport pandas as pd\nimport numpy as np\nfrom nltk.corpus import stopwords\nimport string\nfrom textblob import TextBlob\nimport time\n\n\nstart = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_data(datadir):\n    train_df = pd.read_csv(os.path.join(datadir, 'train.csv'))   \n    test_df = pd.read_csv(os.path.join(datadir, 'test.csv'))   \n    print(\"Train shape : \", train_df.shape)\n    print(\"Test shape : \", test_df.shape)\n    return train_df, test_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_features(df):\n    print('Adding some features to data')\n    \n    ## Transform it to string\n    df[\"question_text\"] = df[\"question_text\"].apply(lambda x: str(x))\n    print('To Sting')\n    ## Number of words in the text ##\n    df[\"num_words\"] = df[\"question_text\"].apply(lambda x: len(x.split()))\n    print('Number of words')\n    ## Number of unique words in the text ##\n    df[\"num_unique_words\"] = df[\"question_text\"].apply(lambda x: len(set(x.split())))\n    print('Number of unique words')\n    ## Number of characters in the text ##\n    df[\"num_chars\"] = df[\"question_text\"].apply(lambda x: len(x))\n    print('Number of chars')\n    ## Number of stopwords in the text ##\n    df[\"num_stopwords\"] = df[\"question_text\"].apply(lambda x: len([w for w in x.lower().split() if w in set(stopwords.words('english'))]))\n    print('Number of Stopwords')\n    ## Number of punctuations in the text ##\n    df[\"num_punctuations\"] =df['question_text'].apply(lambda x: len([c for c in x if c in string.punctuation]) )\n    print('Number of punctuations')\n    ## Number of upper case words in the text ##\n    df[\"num_words_upper\"] = df[\"question_text\"].apply(lambda x: len([w for w in x.split() if w.isupper()]))\n    print('Number of upper words')\n    ## Number of title case words in the text ##\n    df[\"num_words_title\"] = df[\"question_text\"].apply(lambda x: len([w for w in x.split() if w.istitle()]))\n    print('Number of title')\n    ## Number of numbers in the text ##\n    df[\"num_num\"] = df[\"question_text\"].apply(lambda x: len([n for n in x.split() if n.isnumeric()]))\n    print('Number of numbers')\n    ## Average length of the words in the text ##\n    df[\"mean_word_len\"] = df[\"question_text\"].apply(lambda x: np.mean([len(w) for w in x.split()]))\n    print('mean len word')\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_embed(file):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))    \n    return embeddings_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(texts):\n    sentences = texts.apply(lambda x: x.split()).values\n    vocab = set()\n    for sentence in sentences:\n        for word in sentence:\n            vocab.add(word)\n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def add_lower(embedding, df_column):\n    vocab = build_vocab(df_column)\n    count = 0\n    for word in vocab:\n        if word in embedding and word.lower() not in embedding:  \n            embedding[word.lower()] = embedding[word]\n            count += 1\n    print(f\"Added {count} words to embedding\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_lower(df):\n    df[\"question_text\"] = df[\"question_text\"].apply(lambda x: x.lower())\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"abbreviations = {\n    \"ain't\": \"is not\",\n    \"aren't\": \"are not\",\n    \"can't\": \"cannot\",\n    \"'cause\": \"because\",\n    \"could've\": \"could have\",\n    \"couldn't\": \"could not\",\n    \"didn't\": \"did not\",\n    \"doesn't\": \"does not\",\n    \"don't\": \"do not\",\n    \"hadn't\": \"had not\",\n    \"hasn't\": \"has not\",\n    \"haven't\": \"have not\",\n    \"he'd\": \"he would\",\n    \"he'll\": \"he will\",\n    \"he's\": \"he is\",\n    \"how'd\": \"how did\",\n    \"how'd'y\": \"how do you\",\n    \"how'll\": \"how will\",\n    \"how's\": \"how is\",\n    \"I'd\": \"I would\",\n    \"I'd've\": \"I would have\",\n    \"I'll\": \"I will\",\n    \"I'll've\": \"I will have\",\n    \"I'm\": \"I am\",\n    \"I've\": \"I have\",\n    \"i'd\": \"i would\",\n    \"i'd've\": \"i would have\",\n    \"i'll\": \"i will\",\n    \"i'll've\": \"i will have\",\n    \"i'm\": \"i am\",\n    \"i've\": \"i have\",\n    \"isn't\": \"is not\",\n    \"it'd\": \"it would\",\n    \"it'd've\": \"it would have\",\n    \"it'll\": \"it will\",\n    \"it'll've\": \"it will have\",\n    \"it's\": \"it is\",\n    \"let's\": \"let us\",\n    \"ma'am\": \"madam\",\n    \"mayn't\": \"may not\",\n    \"might've\": \"might have\",\n    \"mightn't\": \"might not\",\n    \"mightn't've\": \"might not have\",\n    \"must've\": \"must have\",\n    \"mustn't\": \"must not\",\n    \"mustn't've\": \"must not have\",\n    \"needn't\": \"need not\",\n    \"needn't've\": \"need not have\",\n    \"o'clock\": \"of the clock\",\n    \"oughtn't\": \"ought not\",\n    \"oughtn't've\": \"ought not have\",\n    \"shan't\": \"shall not\",\n    \"sha'n't\": \"shall not\",\n    \"shan't've\": \"shall not have\",\n    \"she'd\": \"she would\",\n    \"she'd've\": \"she would have\",\n    \"she'll\": \"she will\",\n    \"she'll've\": \"she will have\",\n    \"she's\": \"she is\",\n    \"should've\": \"should have\",\n    \"shouldn't\": \"should not\",\n    \"shouldn't've\": \"should not have\",\n    \"so've\": \"so have\",\n    \"so's\": \"so as\",\n    \"this's\": \"this is\",\n    \"that'd\": \"that would\",\n    \"that'd've\": \"that would have\",\n    \"that's\": \"that is\",\n    \"there'd\": \"there would\",\n    \"there'd've\": \"there would have\",\n    \"there's\": \"there is\",\n    \"here's\": \"here is\",\n    \"they'd\": \"they would\",\n    \"they'd've\": \"they would have\",\n    \"they'll\": \"they will\",\n    \"they'll've\": \"they will have\",\n    \"they're\": \"they are\",\n    \"they've\": \"they have\",\n    \"to've\": \"to have\",\n    \"wasn't\": \"was not\",\n    \"we'd\": \"we would\",\n    \"we'd've\": \"we would have\",\n    \"we'll\": \"we will\",\n    \"we'll've\": \"we will have\",\n    \"we're\": \"we are\",\n    \"we've\": \"we have\",\n    \"weren't\": \"were not\",\n    \"what'll\": \"what will\",\n    \"what'll've\": \"what will have\",\n    \"what're\": \"what are\",\n    \"what's\": \"what is\",\n    \"what've\": \"what have\",\n    \"when's\": \"when is\",\n    \"when've\": \"when have\",\n    \"where'd\": \"where did\",\n    \"where's\": \"where is\",\n    \"where've\": \"where have\",\n    \"who'll\": \"who will\",\n    \"who'll've\": \"who will have\",\n    \"who's\": \"who is\",\n    \"who've\": \"who have\",\n    \"why's\": \"why is\",\n    \"why've\": \"why have\",\n    \"will've\": \"will have\",\n    \"won't\": \"will not\",\n    \"won't've\": \"will not have\",\n    \"would've\": \"would have\",\n    \"wouldn't\": \"would not\",\n    \"wouldn't've\": \"would not have\",\n    \"y'all\": \"you all\",\n    \"y'all'd\": \"you all would\",\n    \"y'all'd've\": \"you all would have\",\n    \"y'all're\": \"you all are\",\n    \"y'all've\": \"you all have\",\n    \"you'd\": \"you would\",\n    \"you'd've\": \"you would have\",\n    \"you'll\": \"you will\",\n    \"you'll've\": \"you will have\",\n    \"you're\": \"you are\",\n    \"you've\": \"you have\",\n    \"who'd\": \"who would\",\n    \"who're\": \"who are\",\n    \"'re\": \" are\",\n    \"tryin'\": \"trying\",\n    \"doesn'\": \"does not\",\n    'howdo': 'how do',\n    'whatare': 'what are',\n    'howcan': 'how can',\n    'howmuch': 'how much',\n    'howmany': 'how many',\n    'whydo': 'why do',\n    'doI': 'do I',\n    'theBest': 'the best',\n    'howdoes': 'how does',\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_abbreviation(df, abbreviations):\n    compiled_abbreviation = re.compile('(%s)' % '|'.join(abbreviations.keys()))\n    def replace(match):\n        return abbreviations[match.group(0)]\n    df['question_text'] = df[\"question_text\"].apply(\n        lambda x: _clean_abreviation(x, compiled_abbreviation, replace)\n    )\n    return df\n    \ndef _clean_abreviation(x, compiled_re, replace):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        x = x.replace(s, \"'\")\n    return compiled_re.sub(replace, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## I added social media to the list\nspells = {\n    'colour': 'color',\n    'centre': 'center',\n    'favourite': 'favorite',\n    'travelling': 'traveling',\n    'counselling': 'counseling',\n    'theatre': 'theater',\n    'cancelled': 'canceled',\n    'labour': 'labor',\n    'organisation': 'organization',\n    'wwii': 'world war 2',\n    'citicise': 'criticize',\n    'youtu.be': 'youtube',\n    'youtu ': 'youtube ',\n    'qoura': 'quora',\n    'sallary': 'salary',\n    'Whta': 'what',\n    'whta': 'what',\n    'narcisist': 'narcissist',\n    'mastrubation': 'masturbation',\n    'mastrubate': 'masturbate',\n    \"mastrubating\": 'masturbating',\n    'pennis': 'penis',\n    'Etherium': 'ethereum',\n    'etherium': 'ethereum',\n    'narcissit': 'narcissist',\n    'bigdata': 'big data',\n    '2k': '2000',\n    '2k10': '2010',\n    '2k11': '2011',\n    '2k12': '2012',\n    '2k13': '2013',\n    '2k14': '2014',\n    '2k15': '2015',\n    '2k16': '2016',\n    '2k17': '2017',\n    '2k18': '2018',\n    'qouta': 'quota',\n    'exboyfriend': 'ex boyfriend',\n    'exgirlfriend': 'ex girlfriend',\n    'airhostess': 'air hostess',\n    \"whst\": 'what',\n    'watsapp': 'whatsapp',\n    'demonitisation': 'demonetization',\n    'demonitization': 'demonetization',\n    'demonetisation': 'demonetization',\n    'quorans': 'quora user',\n    'quoran': 'quora user',\n    'pokémon': 'pokemon',\n    'instagram': 'social medium',\n    'whatsapp': 'social medium',\n    'snapchat': 'social medium',\n    'pubg': 'video game',\n    'dota': 'video game',\n    'dota2': 'video game',\n    'fortnite': 'video game',\n    'league of legends': 'video game'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_spells(df, spells):\n    compiled_spells = re.compile('(%s)' % '|'.join(spells.keys()))\n    def replace(match):\n        return spells[match.group(0)]\n    df['question_text'] = df[\"question_text\"].apply(\n        lambda x: _clean_spells(x, compiled_spells, replace)\n    )\n    return df\n\ndef _clean_spells(x, compiled_re, replace):\n    return compiled_re.sub(replace, x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_numbers(df):  \n    df['question_text'] = df['question_text'].apply(lambda x: _clean_numbers(x))\n    return df\n\n\ndef _clean_numbers(x):\n    if bool(re.search(r'\\d', x)):\n        x = re.sub('[0-9]{5,}', '#####', x)\n        x = re.sub('[0-9]{4}', '####', x)\n        x = re.sub('[0-9]{3}', '###', x)\n        x = re.sub('[0-9]{2}', '##', x)\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## I removed '#' sign from the list\nall_puncts={',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '*', '+', '\\\\', \n        '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', \n        '█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', \n        '—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞', \n        '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━', \n        '⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○', \n        '◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐', \n        '├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\\x97', '↺', \n        '∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊', \n        '┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´', \n        '↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\\x93', '≧', '］', \n        '\\x94', '∀', '♛', '\\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％', \n        '╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭', \n        '┊', '╯', '☾', '➔', '∴', '\\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐', \n        '▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\\x91', '➚', '◡', '╰', '\\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒', \n        '☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜', \n        '‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢', \n        '≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '⎯', '↠', '۩', '☰', '◥', \n        '⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ', \n        '✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦', \n        '◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ', \n        '⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋', \n        '､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞', \n        '◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ', \n        '✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱', \n        '☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', \n        '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹', \n        '⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚', \n        '☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛', \n        '⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙', \n        'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘', \n        '˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛', \n        '☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑', \n        '✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ', \n        '⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘', \n        '⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊', \n        '⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷', \n        '◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝', \n        '⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢', \n        '≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢'}\n\n\n## checked for GLOVE embedding\nglove_puncs = set(\"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&')\n\n## I'll fix these puncs using a dict\npunct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi'}\n        \n## Remove other puncs\nremove_punc = all_puncts - glove_puncs\n\n## Remove useless variables\ndel all_puncts\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_and_map_special_chars(df, punct, mapping, removes):\n    df[\"question_text\"] = df[\"question_text\"].apply(lambda x: _clean_and_map_special_chars(x, punct, mapping, removes))\n    return df\n\ndef _clean_and_map_special_chars(text, punct, mapping, removes):\n    for r in removes:\n        if r in text:\n            text = text.replace(r, '')\n        \n    for p in punct:\n        if p in text:\n            text = text.replace(p, f' {p} ')\n        \n    for m in mapping:\n        if m in text:\n            text = text.replace(m, mapping[m])\n     \n    specials = {'\\u200b': ' ', '…': '', '\\ufeff': '', 'करना': '', 'है': '',\n               '\\x7f':'', '\\xa0':'', '\\ufeff':'', '\\u200e':'', '\\u202a':'',\n                '\\u202c':'', '\\u2060':'', '\\uf0d8':'', '\\ue019':'', '\\uf02d':'',\n                '\\u200f':'', '\\u2061':'', '\\ue01b':'', '\\n':' ', '\\t':' ' }  # Other special characters that I have to deal with in last\n    \n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_space(df):\n    compiled_re = re.compile(r\"\\s+\")\n    df['question_text'] = df[\"question_text\"].apply(lambda x: _clean_space(x, compiled_re))\n    return df\n\ndef _clean_space(x, compiled_re):\n    return compiled_re.sub(\" \", x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean(df):\n    df = clean_lower(df)\n    print('Lower is done')\n    df = clean_abbreviation(df, abbreviations)\n    print('Abbreviation is done')\n    df = clean_spells(df, spells)\n    print('Spells is done')\n    df = clean_and_map_special_chars(df, glove_puncs, punct_mapping, remove_punc)\n    print('Special Chars is done')\n    df = clean_numbers(df)\n    print('Remove numbers is done')\n    df = clean_space(df)\n    print('Spaces is done')\n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"datadir = '../input/quora-insincere-questions-classification'\ntrain_df, test_df = load_data(datadir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"glovedir = os.path.join('../input/quora-insincere-questions-classification', 'embeddings', 'glove.840B.300d', 'glove.840B.300d.txt')\nembed_glove = load_embed(glovedir)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"add_lower(embed_glove,\n          pd.concat((train_df['question_text'], test_df['question_text']),axis=0, ignore_index=True))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_df = add_features(train_df)\n# test_df = add_features(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = clean(train_df)\ntest_df = clean(test_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## split to train and val\nfrom sklearn.model_selection import train_test_split\n\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2019)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n## Some parameters\nembed_size = 300 # how big is each word vector\nmax_features = 85000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 64 # max number of words in a question\n\n## Fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Word index\nword_index = tokenizer.word_index\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef make_embed_matrix(embeddings_index, word_index, max_features):\n    \n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size+2))\n    \n    for word, i in word_index.items():\n        if i < max_features:\n            embedding_vector = embeddings_index.get(word)\n            word_sent = TextBlob(word).sentiment\n            ## Extra information we are passing to our embeddings\n            extra_embed = [word_sent.polarity,word_sent.subjectivity]\n            if embedding_vector is not None:\n                embedding_matrix[i] = np.append(embedding_vector,extra_embed)\n                \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix = make_embed_matrix(embed_glove, word_index, max_features)\ndel word_index\ndel embed_glove\n# del train_df\n# del val_df\n# del test_df\ndel tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    \n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\n\ninp = tf.keras.layers.Input(shape=(maxlen,))\nx = tf.keras.layers.Embedding(max_features, embed_size+2, weights=[embedding_matrix], trainable=False)(inp)\nx = tf.keras.layers.SpatialDropout1D(0.3)(x)\nx1 = tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNLSTM(128, return_sequences=True))(x)\nx2 = tf.keras.layers.Bidirectional(tf.compat.v1.keras.layers.CuDNNGRU(64, return_sequences=True))(x1)\nmax_pl = tf.keras.layers.GlobalMaxPooling1D()(x1)\navg_pl = tf.keras.layers.GlobalMaxPooling1D()(x2)\nx = tf.compat.v1.keras.layers.concatenate([max_pl, avg_pl])\n# x = tf.keras.layers.Dense(32, activation=\"relu\")(x)\n# x = tf.keras.layers.Dropout(0.2)(x)\nx = tf.keras.layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = tf.keras.models.Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer=tf.compat.v1.keras.optimizers.Adam(), metrics=['accuracy', f1])\nprint(model.summary())\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 7\nbatch_size = 512\n\n# from keras.callbacks import ModelCheckpoint, ReduceLROnPlateau\ncheckpoints_treated = tf.keras.callbacks.ModelCheckpoint('weights.hdf5', monitor=\"val_loss\", mode=\"min\", verbose=True, save_best_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history = model.fit(train_X, train_y, batch_size=batch_size, epochs=epochs, \n                            validation_data=[val_X, val_y], callbacks=[checkpoints_treated,])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# plt.figure(figsize=(12,8))\n# plt.plot(history.history['loss'], label='Train Accuracy')\n# plt.plot(history.history['val_loss'], label='Test Accuracy')\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.load_weights('weights.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val = model.predict([val_X], batch_size=512, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import f1_score\nbest_score = 0\nbest_thresh = 0\nfor thresh in np.arange(0.1, 0.501, 0.001):\n    thresh = np.round(thresh, 3)\n    temp_score = f1_score(val_y, (pred_val>thresh).astype(int))\n    if temp_score >= best_score:\n        best_score = temp_score\n        best_thresh = thresh","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print( best_score, best_thresh)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test_y = model.predict([test_X], batch_size=512, verbose=0)\npred_test_y = (pred_test_y>best_thresh).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)\n\nend=time.time()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.7.5"}},"nbformat":4,"nbformat_minor":1}