{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n**This notebook introduced how to solve an imbalanced text classification problem with LSTM networks and word embedding.**","metadata":{}},{"cell_type":"markdown","source":"Import some required libraries.","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\n%matplotlib inline\n\nimport gc\nimport sys\n\nfrom tqdm.notebook import tqdm\ntqdm().pandas()\npd.set_option('display.max_colwidth', None)\n\n# Set seed for experiment reproducibility\nseed = 1024\ntf.random.set_seed(seed)\nnp.random.seed(seed)\n\ndef print_size(var):  \n    print('%.2fMB' % (sys.getsizeof(var)/1024/1024))","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Unzip the word2vec. It may take several minutes.","metadata":{}},{"cell_type":"code","source":"GLOVE_FILE = 'glove.840B.300d/glove.840B.300d.txt'","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip -n /kaggle/input/quora-insincere-questions-classification/embeddings.zip {GLOVE_FILE} -d .","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_coefs(word, *arr): return word, np.asarray(arr, dtype='float32')\ndef get_lines_count(file_name): return sum(1 for _ in open(file_name, encoding=\"utf8\", errors='ignore'))\ndef load_vec(file_name): return dict(get_coefs(*o.split(\" \")) for o in tqdm(open(file_name, encoding=\"utf8\", errors='ignore'), total=get_lines_count(file_name)) if len(o) > 100)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Load the train and test dataset.","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ntest_data = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"It's not necessary using entire dataset to train if you just run it quickly.","metadata":{}},{"cell_type":"code","source":"# train_data = train_data[0:100000]\n# test_data = test_data[0:10000]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see what's in the dataset and print the first 5 rows in train data.","metadata":{}},{"cell_type":"code","source":"train_data.head()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's see how imbalanced the dataset is.","metadata":{}},{"cell_type":"code","source":"negative, positive = np.bincount(train_data['target'])\ntotal = negative + positive\nprint('total: {}    positive: {} ({:.2f}% of total)'.format(total, positive, 100 * positive / total))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Word vectorizing. converting words into numbers so that can be fed into neural network.","metadata":{}},{"cell_type":"code","source":"import re\n\ndef clean_tag(text):\n    if '[math]' in text:\n        text = re.sub('\\[math\\].*?math\\]', '[formula]', text) #replacing with [formuala]\n\n    if 'http' in text or 'www' in text:\n        text = re.sub('(?:(?:https?|ftp):\\/\\/)?[\\w/\\-?=%.]+\\.[\\w/\\-?=%.]+', '[url]', text) #replacing with [url]\n    return text\n\ncontraction_mapping = {\"We'd\": \"We had\", \"That'd\": \"That had\", \"AREN'T\": \"Are not\", \"HADN'T\": \"Had not\", \"Could've\": \"Could have\", \"LeT's\": \"Let us\", \"How'll\": \"How will\", \"They'll\": \"They will\", \"DOESN'T\": \"Does not\", \"HE'S\": \"He has\", \"O'Clock\": \"Of the clock\", \"Who'll\": \"Who will\", \"What'S\": \"What is\", \"Ain't\": \"Am not\", \"WEREN'T\": \"Were not\", \"Y'all\": \"You all\", \"Y'ALL\": \"You all\", \"Here's\": \"Here is\", \"It'd\": \"It had\", \"Should've\": \"Should have\", \"I'M\": \"I am\", \"ISN'T\": \"Is not\", \"Would've\": \"Would have\", \"He'll\": \"He will\", \"DON'T\": \"Do not\", \"She'd\": \"She had\", \"WOULDN'T\": \"Would not\", \"She'll\": \"She will\", \"IT's\": \"It is\", \"There'd\": \"There had\", \"It'll\": \"It will\", \"You'll\": \"You will\", \"He'd\": \"He had\", \"What'll\": \"What will\", \"Ma'am\": \"Madam\", \"CAN'T\": \"Can not\", \"THAT'S\": \"That is\", \"You've\": \"You have\", \"She's\": \"She is\", \"Weren't\": \"Were not\", \"They've\": \"They have\", \"Couldn't\": \"Could not\", \"When's\": \"When is\", \"Haven't\": \"Have not\", \"We'll\": \"We will\", \"That's\": \"That is\", \"We're\": \"We are\", \"They're\": \"They' are\", \"You'd\": \"You would\", \"How'd\": \"How did\", \"What're\": \"What are\", \"Hasn't\": \"Has not\", \"Wasn't\": \"Was not\", \"Won't\": \"Will not\", \"There's\": \"There is\", \"Didn't\": \"Did not\", \"Doesn't\": \"Does not\", \"You're\": \"You are\", \"He's\": \"He is\", \"SO's\": \"So is\", \"We've\": \"We have\", \"Who's\": \"Who is\", \"Wouldn't\": \"Would not\", \"Why's\": \"Why is\", \"WHO's\": \"Who is\", \"Let's\": \"Let us\", \"How's\": \"How is\", \"Can't\": \"Can not\", \"Where's\": \"Where is\", \"They'd\": \"They had\", \"Don't\": \"Do not\", \"Shouldn't\":\"Should not\", \"Aren't\":\"Are not\", \"ain't\": \"is not\", \"What's\": \"What is\", \"It's\": \"It is\", \"Isn't\":\"Is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }\n\ndef clean_contractions(text):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    \n    text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(\" \")])\n    return text\n\npuncts = [\",\",\".\",'\"',\":\",\")\",\"(\",\"-\",\"!\",\"?\",\"|\",\";\",\"'\",\"$\",\"&\",\"/\",\"[\",\"]\",\">\",\"%\",\"=\",\"#\",\"*\",\"+\",\"\\\\\",\"•\",\"~\",\"@\",\"£\",\"·\",\"_\",\"{\",\"}\",\"©\",\"^\",\"®\",\"`\",\"<\",\"→\",\"°\",\"€\",\"™\",\"›\",\"♥\",\"←\",\"×\",\"§\",\"″\",\"′\",\"█\",\"…\",\"“\",\"★\",\"”\",\"–\",\"●\",\"►\",\"−\",\"¢\",\"¬\",\"░\",\"¡\",\"¶\",\"↑\",\"±\",\"¿\",\"▾\",\"═\",\"¦\",\"║\",\"―\",\"¥\",\"▓\",\"—\",\"‹\",\"─\",\"▒\",\"：\",\"⊕\",\"▼\",\"▪\",\"†\",\"■\",\"’\",\"▀\",\"¨\",\"▄\",\"♫\",\"☆\",\"¯\",\"♦\",\"¤\",\"▲\",\"¸\",\"⋅\",\"‘\",\"∞\",\"∙\",\"）\",\"↓\",\"、\",\"│\",\"（\",\"»\",\"，\",\"♪\",\"╩\",\"╚\",\"・\",\"╦\",\"╣\",\"╔\",\"╗\",\"▬\",\"❤\",\"≤\",\"‡\",\"√\",\"◄\",\"━\",\"⇒\",\"▶\",\"≥\",\"╝\",\"♡\",\"◊\",\"。\",\"✈\",\"≡\",\"☺\",\"✔\",\"↵\",\"≈\",\"✓\",\"♣\",\"☎\",\"℃\",\"◦\",\"└\",\"‟\",\"～\",\"！\",\"○\",\"◆\",\"№\",\"♠\",\"▌\",\"✿\",\"▸\",\"⁄\",\"□\",\"❖\",\"✦\",\"．\",\"÷\",\"｜\",\"┃\",\"／\",\"￥\",\"╠\",\"↩\",\"✭\",\"▐\",\"☼\",\"☻\",\"┐\",\"├\",\"«\",\"∼\",\"┌\",\"℉\",\"☮\",\"฿\",\"≦\",\"♬\",\"✧\",\"〉\",\"－\",\"⌂\",\"✖\",\"･\",\"◕\",\"※\",\"‖\",\"◀\",\"‰\",\"\\x97\",\"↺\",\"∆\",\"┘\",\"┬\",\"╬\",\"،\",\"⌘\",\"⊂\",\"＞\",\"〈\",\"⎙\",\"？\",\"☠\",\"⇐\",\"▫\",\"∗\",\"∈\",\"≠\",\"♀\",\"♔\",\"˚\",\"℗\",\"┗\",\"＊\",\"┼\",\"❀\",\"＆\",\"∩\",\"♂\",\"‿\",\"∑\",\"‣\",\"➜\",\"┛\",\"⇓\",\"☯\",\"⊖\",\"☀\",\"┳\",\"；\",\"∇\",\"⇑\",\"✰\",\"◇\",\"♯\",\"☞\",\"´\",\"↔\",\"┏\",\"｡\",\"◘\",\"∂\",\"✌\",\"♭\",\"┣\",\"┴\",\"┓\",\"✨\",\"\\xa0\",\"˜\",\"❥\",\"┫\",\"℠\",\"✒\",\"［\",\"∫\",\"\\x93\",\"≧\",\"］\",\"\\x94\",\"∀\",\"♛\",\"\\x96\",\"∨\",\"◎\",\"↻\",\"⇩\",\"＜\",\"≫\",\"✩\",\"✪\",\"♕\",\"؟\",\"₤\",\"☛\",\"╮\",\"␊\",\"＋\",\"┈\",\"％\",\"╋\",\"▽\",\"⇨\",\"┻\",\"⊗\",\"￡\",\"।\",\"▂\",\"✯\",\"▇\",\"＿\",\"➤\",\"✞\",\"＝\",\"▷\",\"△\",\"◙\",\"▅\",\"✝\",\"∧\",\"␉\",\"☭\",\"┊\",\"╯\",\"☾\",\"➔\",\"∴\",\"\\x92\",\"▃\",\"↳\",\"＾\",\"׳\",\"➢\",\"╭\",\"➡\",\"＠\",\"⊙\",\"☢\",\"˝\",\"∏\",\"„\",\"∥\",\"❝\",\"☐\",\"▆\",\"╱\",\"⋙\",\"๏\",\"☁\",\"⇔\",\"▔\",\"\\x91\",\"➚\",\"◡\",\"╰\",\"\\x85\",\"♢\",\"˙\",\"۞\",\"✘\",\"✮\",\"☑\",\"⋆\",\"ⓘ\",\"❒\",\"☣\",\"✉\",\"⌊\",\"➠\",\"∣\",\"❑\",\"◢\",\"ⓒ\",\"\\x80\",\"〒\",\"∕\",\"▮\",\"⦿\",\"✫\",\"✚\",\"⋯\",\"♩\",\"☂\",\"❞\",\"‗\",\"܂\",\"☜\",\"‾\",\"✜\",\"╲\",\"∘\",\"⟩\",\"＼\",\"⟨\",\"·\",\"✗\",\"♚\",\"∅\",\"ⓔ\",\"◣\",\"͡\",\"‛\",\"❦\",\"◠\",\"✄\",\"❄\",\"∃\",\"␣\",\"≪\",\"｢\",\"≅\",\"◯\",\"☽\",\"∎\",\"｣\",\"❧\",\"̅\",\"ⓐ\",\"↘\",\"⚓\",\"▣\",\"˘\",\"∪\",\"⇢\",\"✍\",\"⊥\",\"＃\",\"⎯\",\"↠\",\"۩\",\"☰\",\"◥\",\"⊆\",\"✽\",\"⚡\",\"↪\",\"❁\",\"☹\",\"◼\",\"☃\",\"◤\",\"❏\",\"ⓢ\",\"⊱\",\"➝\",\"̣\",\"✡\",\"∠\",\"｀\",\"▴\",\"┤\",\"∝\",\"♏\",\"ⓐ\",\"✎\",\";\",\"␤\",\"＇\",\"❣\",\"✂\",\"✤\",\"ⓞ\",\"☪\",\"✴\",\"⌒\",\"˛\",\"♒\",\"＄\",\"✶\",\"▻\",\"ⓔ\",\"◌\",\"◈\",\"❚\",\"❂\",\"￦\",\"◉\",\"╜\",\"̃\",\"✱\",\"╖\",\"❉\",\"ⓡ\",\"↗\",\"ⓣ\",\"♻\",\"➽\",\"׀\",\"✲\",\"✬\",\"☉\",\"▉\",\"≒\",\"☥\",\"⌐\",\"♨\",\"✕\",\"ⓝ\",\"⊰\",\"❘\",\"＂\",\"⇧\",\"̵\",\"➪\",\"▁\",\"▏\",\"⊃\",\"ⓛ\",\"‚\",\"♰\",\"́\",\"✏\",\"⏑\",\"̶\",\"ⓢ\",\"⩾\",\"￠\",\"❍\",\"≃\",\"⋰\",\"♋\",\"､\",\"̂\",\"❋\",\"✳\",\"ⓤ\",\"╤\",\"▕\",\"⌣\",\"✸\",\"℮\",\"⁺\",\"▨\",\"╨\",\"ⓥ\",\"♈\",\"❃\",\"☝\",\"✻\",\"⊇\",\"≻\",\"♘\",\"♞\",\"◂\",\"✟\",\"⌠\",\"✠\",\"☚\",\"✥\",\"❊\",\"ⓒ\",\"⌈\",\"❅\",\"ⓡ\",\"♧\",\"ⓞ\",\"▭\",\"❱\",\"ⓣ\",\"∟\",\"☕\",\"♺\",\"∵\",\"⍝\",\"ⓑ\",\"✵\",\"✣\",\"٭\",\"♆\",\"ⓘ\",\"∶\",\"⚜\",\"◞\",\"்\",\"✹\",\"➥\",\"↕\",\"̳\",\"∷\",\"✋\",\"➧\",\"∋\",\"̿\",\"ͧ\",\"┅\",\"⥤\",\"⬆\",\"⋱\",\"☄\",\"↖\",\"⋮\",\"۔\",\"♌\",\"ⓛ\",\"╕\",\"♓\",\"❯\",\"♍\",\"▋\",\"✺\",\"⭐\",\"✾\",\"♊\",\"➣\",\"▿\",\"ⓑ\",\"♉\",\"⏠\",\"◾\",\"▹\",\"⩽\",\"↦\",\"╥\",\"⍵\",\"⌋\",\"։\",\"➨\",\"∮\",\"⇥\",\"ⓗ\",\"ⓓ\",\"⁻\",\"⎝\",\"⌥\",\"⌉\",\"◔\",\"◑\",\"✼\",\"♎\",\"♐\",\"╪\",\"⊚\",\"☒\",\"⇤\",\"ⓜ\",\"⎠\",\"◐\",\"⚠\",\"╞\",\"◗\",\"⎕\",\"ⓨ\",\"☟\",\"ⓟ\",\"♟\",\"❈\",\"↬\",\"ⓓ\",\"◻\",\"♮\",\"❙\",\"♤\",\"∉\",\"؛\",\"⁂\",\"ⓝ\",\"־\",\"♑\",\"╫\",\"╓\",\"╳\",\"⬅\",\"☔\",\"☸\",\"┄\",\"╧\",\"׃\",\"⎢\",\"❆\",\"⋄\",\"⚫\",\"̏\",\"☏\",\"➞\",\"͂\",\"␙\",\"ⓤ\",\"◟\",\"̊\",\"⚐\",\"✙\",\"↙\",\"̾\",\"℘\",\"✷\",\"⍺\",\"❌\",\"⊢\",\"▵\",\"✅\",\"ⓖ\",\"☨\",\"▰\",\"╡\",\"ⓜ\",\"☤\",\"∽\",\"╘\",\"˹\",\"↨\",\"♙\",\"⬇\",\"♱\",\"⌡\",\"⠀\",\"╛\",\"❕\",\"┉\",\"ⓟ\",\"̀\",\"♖\",\"ⓚ\",\"┆\",\"⎜\",\"◜\",\"⚾\",\"⤴\",\"✇\",\"╟\",\"⎛\",\"☩\",\"➲\",\"➟\",\"ⓥ\",\"ⓗ\",\"⏝\",\"◃\",\"╢\",\"↯\",\"✆\",\"˃\",\"⍴\",\"❇\",\"⚽\",\"╒\",\"̸\",\"♜\",\"☓\",\"➳\",\"⇄\",\"☬\",\"⚑\",\"✐\",\"⌃\",\"◅\",\"▢\",\"❐\",\"∊\",\"☈\",\"॥\",\"⎮\",\"▩\",\"ு\",\"⊹\",\"‵\",\"␔\",\"☊\",\"➸\",\"̌\",\"☿\",\"⇉\",\"⊳\",\"╙\",\"ⓦ\",\"⇣\",\"｛\",\"̄\",\"↝\",\"⎟\",\"▍\",\"❗\",\"״\",\"΄\",\"▞\",\"◁\",\"⛄\",\"⇝\",\"⎪\",\"♁\",\"⇠\",\"☇\",\"✊\",\"ி\",\"｝\",\"⭕\",\"➘\",\"⁀\",\"☙\",\"❛\",\"❓\",\"⟲\",\"⇀\",\"≲\",\"ⓕ\",\"⎥\",\"\\u06dd\",\"ͤ\",\"₋\",\"̱\",\"̎\",\"♝\",\"≳\",\"▙\",\"➭\",\"܀\",\"ⓖ\",\"⇛\",\"▊\",\"⇗\",\"̷\",\"⇱\",\"℅\",\"ⓧ\",\"⚛\",\"̐\",\"̕\",\"⇌\",\"␀\",\"≌\",\"ⓦ\",\"⊤\",\"̓\",\"☦\",\"ⓕ\",\"▜\",\"➙\",\"ⓨ\",\"⌨\",\"◮\",\"☷\",\"◍\",\"ⓚ\",\"≔\",\"⏩\",\"⍳\",\"℞\",\"┋\",\"˻\",\"▚\",\"≺\",\"ْ\",\"▟\",\"➻\",\"̪\",\"⏪\",\"̉\",\"⎞\",\"┇\",\"⍟\",\"⇪\",\"▎\",\"⇦\",\"␝\",\"⤷\",\"≖\",\"⟶\",\"♗\",\"̴\",\"♄\",\"ͨ\",\"̈\",\"❜\",\"̡\",\"▛\",\"✁\",\"➩\",\"ா\",\"˂\",\"↥\",\"⏎\",\"⎷\",\"̲\",\"➖\",\"↲\",\"⩵\",\"̗\",\"❢\",\"≎\",\"⚔\",\"⇇\",\"̑\",\"⊿\",\"̖\",\"☍\",\"➹\",\"⥊\",\"⁁\",\"✢\"];\n\ndef clean_punct(x):\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\ndef data_cleaning(x):\n    x = clean_tag(x)\n    x = clean_contractions(x)\n    x = clean_punct(x)\n    return x\n\ntrain_data['preprocessed_question_text'] = train_data['question_text'].progress_map(lambda x: data_cleaning(x))\ntest_data['preprocessed_question_text'] = test_data['question_text'].progress_map(lambda x: data_cleaning(x))","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow import keras\nimport spacy\n\nnlp = spacy.load('en_core_web_lg', disable=['parser','ner','tagger'])\n\nvocab_freq = {}\nword2index = {}\nlemma_dict = {}\n\nsentences = pd.concat([train_data[\"preprocessed_question_text\"], test_data[\"preprocessed_question_text\"]])\ndocs = nlp.pipe(sentences, n_threads = 2)\nword_sequences = []\n\nfor doc in tqdm(docs, total=len(sentences)):\n    word_seq = []\n    for token in doc:\n        if token.is_punct or token.is_space:\n            continue\n        try:\n            vocab_freq[token.text] += 1\n        except KeyError:\n            vocab_freq[token.text] = 1\n        if token.text not in word2index:\n            word2index[token.text] = len(vocab_freq)\n            lemma_dict[token.text] = token.lemma_\n        word_seq.append(word2index[token.text])\n    word_sequences.append(word_seq)\n\nvocab_size = len(word2index) + 1\n\nprint('Found %s unique tokens.' % vocab_size)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define the max sentence length. The length should be longer than most sentences in the dataset, otherwise it will lose a lot of useful features.","metadata":{}},{"cell_type":"code","source":"MAX_SENTENCE_LENGTH = 100\n\nmax_text_len = len(max(word_sequences, key=len))\nprint(\"max text length in data: \", max_text_len)\n\npercentage = 100 * sum(1 for seq in word_sequences if len(seq) > MAX_SENTENCE_LENGTH)/total\nprint(\"percentage of sentences that's length longer than max length > %d in data: %.4f%%\" % (MAX_SENTENCE_LENGTH, percentage))\n\nX_data = word_sequences[:len(train_data)]\nX_data = keras.preprocessing.sequence.pad_sequences(X_data, maxlen=MAX_SENTENCE_LENGTH)\nprint('Shape of data tensor:', X_data.shape)\n\nX_test_data = word_sequences[len(train_data):]\nX_test_data = keras.preprocessing.sequence.pad_sequences(X_test_data, maxlen=MAX_SENTENCE_LENGTH)\n\nY_data = train_data['target']\n\ndel word_sequences, docs, sentences, train_data\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Fill the embedding layer's weights with word2vec we had loaded before.","metadata":{}},{"cell_type":"code","source":"from nltk.stem import PorterStemmer, SnowballStemmer, WordNetLemmatizer\nfrom nltk.stem.lancaster import LancasterStemmer\n \nEMBEDDING_DIM = 300\n\nps = PorterStemmer()\nlc = LancasterStemmer()\nsb = SnowballStemmer('english')\nlm = WordNetLemmatizer() \n\ndef correction(word):\n    return list(candidates(word))[0]\n\ndef candidates(word):\n    \"Generate possible spelling corrections for word.\"\n    return (known([word]) or known(edits1(word)) or [word])\n\ndef known(words):\n    \"The subset of `words` that appear in the dictionary of WORDS.\"\n    return set(w for w in words if w in word2index)\n\ndef edits1(word):\n    \"All edits that are one edit away from `word`.\"\n    letters = 'abcdefghijklmnopqrstuvwxyz'\n    splits = [(word[:i], word[i:])        for i in range(len(word) + 1)]\n    deletes = [L + R[1:]                  for L, R in splits if R]  \n    transposes = [L + R[1] + R[0] + R[2:] for L, R in splits if len(R)>1]  \n    replaces = [L + c + R[1:]             for L, R in splits if R for c in letters]\n    inserts = [L + c + R                  for L, R in splits for c in letters]\n    return set(deletes + transposes + replaces + inserts)\n\ndef load_embedding(word2vec):\n    oov_count = 0\n    vocab_count = 0\n    embedding_weights = np.zeros((vocab_size, EMBEDDING_DIM))\n    unknown_vector = np.zeros((EMBEDDING_DIM,), dtype=np.float32) - 1.\n    unknown_words = {}\n\n    for key, i in tqdm(word2index.items()):\n        word = key\n        if word in word2vec:\n            vocab_count += vocab_freq[key]\n            embedding_weights[i] = word2vec[word]\n            continue\n        word = key.lower()         #Lower\n        if word in word2vec:\n            vocab_count += vocab_freq[key]\n            embedding_weights[i] = word2vec[word]\n            continue\n        word = key.upper()         #Upper\n        if word in word2vec:\n            vocab_count += vocab_freq[key]\n            embedding_weights[i] = word2vec[word]\n            continue\n        word = key.capitalize()    #Capitalize \n        if word in word2vec:\n            vocab_count += vocab_freq[key]\n            embedding_weights[i] = word2vec[word]\n            continue\n        word = ps.stem(key)        #PorterStemmer\n        if word in word2vec:\n            vocab_count += vocab_freq[key]\n            embedding_weights[i] = word2vec[word]\n            continue\n        word = lc.stem(key)        #LancasterStemmer\n        if word in word2vec:\n            vocab_count += vocab_freq[key]\n            embedding_weights[i] = word2vec[word]\n            continue\n        word = sb.stem(key)        #SnowballStemmer\n        if word in word2vec:\n            vocab_count += vocab_freq[key]\n            embedding_weights[i] = word2vec[word]\n            continue\n        word = lemma_dict[key]     #Lemmanization\n        if word in word2vec: \n            vocab_count += vocab_freq[key]\n            embedding_weights[i] = word2vec[word]\n            continue\n        if len(key) > 1:\n            word = correction(key)\n            if word in word2vec: \n                vocab_count += vocab_freq[key]\n                embedding_weights[i] = word2vec[word]\n                continue\n\n        try:\n            unknown_words[key] += 1\n        except KeyError:\n            unknown_words[key] = 1\n            \n        embedding_weights[i] = unknown_vector\n        oov_count += vocab_freq[key]\n\n    print('Top 10 Null word embeddings: ')\n    print(list(unknown_words.items())[:10])\n    print('\\n')\n    print('Null word embeddings: %d' % np.sum(np.sum(embedding_weights, axis=1) == -1 * EMBEDDING_DIM))\n    print('Null word embeddings percentage: %.2f%%' % (100 * oov_count / vocab_count))\n    \n    return embedding_weights","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('loading glove_vec')\nglove_vec = load_vec(GLOVE_FILE)\nglove_weights = load_embedding(glove_vec)\ndel glove_vec\ngc.collect()","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Define model. I usually use metrics like f1-score, auc for binary classification model. There has three layers in the model. The first layer is Embedding layer that turns the X_train data(now that's the word indexes of vocabulary) into EMBEDDING_DIM dimensional vectors. The second layer is a bidirectinal LSTM that is well-suited to process data base on time-series. The last layer is output layer with a sigmoid activation function. I had set the [bias_initializer](http://https://www.tensorflow.org/tutorials/structured_data/imbalanced_data#optional_set_the_correct_initial_bias) parameter for the output layer due to the imbalanced dataset.","metadata":{}},{"cell_type":"code","source":"import tensorflow as tf\n\nstrategy = None\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print('Use TPU')\nexcept ValueError:\n    if len(tf.config.list_physical_devices('GPU')) > 0:\n        strategy = tf.distribute.MirroredStrategy()\n        print('Use GPU')\n    else:\n        strategy = tf.distribute.get_strategy()\n        print('Use CPU')","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def f1_smart(y_true, y_pred):\n    args = np.argsort(y_pred)\n    tp = y_true.sum()\n    fs = (tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)\n    res_idx = np.argmax(fs)\n    return 2 * fs[res_idx], (y_pred[args[res_idx]] + y_pred[args[res_idx + 1]]) / 2\n\ndef plot_history(history):\n    plt.plot(history.history['loss'])\n    plt.plot(history.history['val_loss'])\n    plt.title('model loss')\n    plt.ylabel('loss')\n    plt.xlabel('epoch')\n    plt.legend(['train', 'val'])\n    plt.show()\n\nweight_for_0 = (1 / negative) * (total) / 2.0 \nweight_for_1 = (1 / positive) * (total) / 2.0\n\nclass_weight = {0: weight_for_0, 1: weight_for_1}\n\nearly_stopping = keras.callbacks.EarlyStopping(\n    monitor='val_loss', \n    verbose=1,\n    patience=1,\n    mode='min',\n    restore_best_weights=True)\n\ncheckpoint = keras.callbacks.ModelCheckpoint('best_model.h5', monitor='val_loss', save_best_only=True, mode='min')\n\nreduce_lr = keras.callbacks.ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1)","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tensorflow.keras import layers\n\nclass TransformerLayer(layers.Layer):\n    def __init__(self, num_heads, hidden_size, dropout_rate=0.1, **kwargs):\n        super(TransformerLayer, self).__init__(**kwargs)\n        self.num_heads = num_heads\n        self.hidden_size = hidden_size\n        self.dropout_rate = dropout_rate\n        \n    def build(self, input_shape):\n        self.att = layers.MultiHeadAttention(num_heads=self.num_heads, key_dim=input_shape[2])\n        self.ffn = keras.Sequential(\n            [layers.Dense(self.hidden_size, activation=\"relu\"), layers.Dense(input_shape[2]),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(self.dropout_rate)\n        self.dropout2 = layers.Dropout(self.dropout_rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs, inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)\n    \n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'num_heads': self.num_heads,\n            'hidden_size': self.hidden_size,\n            'dropout_rate': self.dropout_rate\n        })\n        return config\n    \nclass PositionEmbedding(layers.Layer):\n    def __init__(self, embeding_dim, **kwargs):\n        super(PositionEmbedding, self).__init__(**kwargs)\n        self.embeding_dim = embeding_dim\n        \n    def build(self, input_shape): \n        self.max_length = input_shape[-1]\n        self.position_embedding = layers.Embedding(input_dim=self.max_length, output_dim=self.embeding_dim, name='position_embedding')\n\n    def call(self, inputs):\n        positions = tf.range(start=0, limit=self.max_length, delta=1)\n        output = self.position_embedding(positions)\n        return output\n    \n    def get_config(self):\n        config = super().get_config().copy()\n        config.update({\n            'embeding_dim': self.embeding_dim\n        })\n        return config","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import tensorflow_addons as tfa\nfrom keras import backend as K\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras.initializers import Constant\n\ndef create_model(units):\n    output_bias = Constant(np.log([positive/negative]))\n    \n    x_input = Input(shape=(MAX_SENTENCE_LENGTH,))\n    posistion_x = PositionEmbedding(EMBEDDING_DIM)(x_input)\n    x = Embedding(vocab_size, EMBEDDING_DIM, input_length=MAX_SENTENCE_LENGTH, weights=[glove_weights], trainable=False)(x_input)\n    x = SpatialDropout1D(0.2)(x)\n    rnn = Bidirectional(GRU(units, return_sequences=True))(x)\n    att = TransformerLayer(2, 128)(posistion_x + x)\n    x = Concatenate()([rnn, att])\n    \n    x = GlobalAveragePooling1D()(x)\n    x_output = Dense(1, activation='sigmoid', bias_initializer=output_bias)(x)\n    \n    model = Model(inputs=x_input, outputs=x_output)\n    model.compile(loss='binary_crossentropy', optimizer='adam')\n    \n    return model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import f1_score\nfrom sklearn.model_selection import StratifiedKFold\nfrom IPython.display import Image\nfrom keras.utils import plot_model\n\nkfold = StratifiedKFold(n_splits=5, shuffle=True, random_state=seed)\n\nbest_f1score = 0\n\nwith strategy.scope():\n    model = create_model(64)\n    model.summary()\n    \n    plot_model(model, 'model.png', show_shapes=True, show_layer_names=True)\n    \n    for index, (train_index, valid_index) in enumerate(kfold.split(X_data, Y_data)):\n        if index > 1:\n            break\n        X_train, X_val, Y_train, Y_val = X_data[train_index], X_data[valid_index], Y_data[train_index], Y_data[valid_index]\n        history = model.fit(\n            X_train, Y_train, \n            epochs=5, \n            batch_size=128, \n            validation_data=(X_val, Y_val),\n            callbacks=[reduce_lr], \n            class_weight=class_weight\n        )\n        plot_history(history)\n        Y_pred = model.predict(X_val)\n        f1, threshold = f1_smart(Y_val.to_numpy(), np.squeeze(Y_pred))\n        best_f1score = max(best_f1score, f1)\n        print('Optimal F1: {:.4f} at threshold: {:.4f}\\n'.format(f1, threshold))\n        \nprint(f'{\"#\" * 30} best f1score: {best_f1score} {\"#\" * 30}')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"Image(\"model.png\")","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Predict on the test dataset and write to the file named as submission.csv.","metadata":{}},{"cell_type":"code","source":"# best_model = keras.models.load_model('best_model.h5', custom_objects={\"TransformerLayer\": TransformerLayer, \"PositionEmbedding\": PositionEmbedding})\n\nY_test = (model.predict(X_test_data) > threshold).astype(\"int32\")\n\nprint('Write results to submission.csv')\nsubmit_data = pd.DataFrame({'qid': test_data.qid, 'prediction': Y_test.reshape(-1)})\nsubmit_data.to_csv('submission.csv', index=False)\n\n!head submission.csv","metadata":{"scrolled":true,"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"References:\n\nhttps://www.kaggle.com/ronaksvijay/qiqc-nn","metadata":{}}]}