{"cells":[{"metadata":{},"cell_type":"markdown","source":"# **Problem**"},{"metadata":{},"cell_type":"markdown","source":"If a question found on Quora is consider as a content appropriet Predictions should only be the integers 0, and if not Predictions should only be the integers 1."},{"metadata":{},"cell_type":"markdown","source":"# **Import Libraries**"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt # librarie qui affiche les graphes\nimport seaborn as sns # Python data visualization library based on matplotlib\nimport re\nimport nltk \nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize.toktok import ToktokTokenizer\nfrom nltk.corpus import stopwords \nfrom nltk.stem import WordNetLemmatizer\nimport string\nfrom string import punctuation\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load Train and Test Data-set"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Reading Data\ndf_train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ndf_test = pd.read_csv('../input/quora-insincere-questions-classification/test.csv') ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# File descriptions\n* train.csv — the training set\n* test.csv — the test set\n* embeddings: glove"},{"metadata":{},"cell_type":"markdown","source":"# Data Fields\n* qid — unique question identifier\n* question_text — Quora question text\n* target — a question labeled “insincere” has a value of 1, otherwise 0"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_train.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(df_test.isnull().sum())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(df_train.question_text[df_train['target'] == 0]) /      len(df_train['question_text']) * 100,'percent of sincere')\nprint(len(df_train.question_text[df_train['target'] == 1]) / len(df_train['question_text']) * 100,'percent of insincere')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"* 93.81298224821265 percent of sincere\n\n* 6.187017751787352 percent of insincere"},{"metadata":{"trusted":true},"cell_type":"code","source":"f, ax = plt.subplots(nrows=1, ncols=1, figsize=(6, 4))\nsns.countplot(df_train.target, palette=['red', 'blue'], ax=ax)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# # 1. DATA Preprocessing (Cleaning)"},{"metadata":{},"cell_type":"markdown","source":"The text data is not entirely clean, thus we need to apply some data preprocessing techniques."},{"metadata":{},"cell_type":"markdown","source":"# Convert questions to lower case"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Convert to lower case\ndef lower1(text):\n    \n    text = text.lower()\n   \n       \n    return text\n\ndf_train[\"question_text\"] = df_train[\"question_text\"].apply(lower1)\ndf_test[\"question_text\"] = df_test[\"question_text\"].apply(lower1)\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Removing Punctuation"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Removing Punctuation\npuncts=[',', '.', '“', ':', ')', '(', '-', '!', '|', ';', '\\'', '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', \n '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', \n '█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', \n ' — ', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '\\'', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '\\'', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━', \n '⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○', \n '◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐', \n '├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\\x97', '↺', \n '∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊', \n '┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´', \n '↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\\x93', '≧', '］', \n '\\x94', '∀', '♛', '\\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％', \n '╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭', \n '┊', '╯', '☾', '➔', '∴', '\\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐', \n '▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\\x91', '➚', '◡', '╰', '\\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒', \n '☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜', \n '‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢', \n '≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥', \n '⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ', \n '✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦', \n '◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ', \n '⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋', \n '､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞', \n '◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ', \n '✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱', \n '☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹', \n '⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚', \n '☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛', \n '⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙', \n 'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘', \n '˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛', \n '☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑', \n '✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ', \n '⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘', \n '⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊', \n '⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷', \n '◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝', \n '⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢', \n '≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']\n\ndef clean_punct(x):\n\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct,'')\n\n    return x\ndf_train[\"question_text\"] = df_train[\"question_text\"].apply(clean_punct)\ndf_test[\"question_text\"] = df_test[\"question_text\"].apply(clean_punct)\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_train.target","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Removing Numbers"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Cleaning numbers\ndef remove_numbers(x): \n    x = str(x)\n    return re.sub(r'\\d+', '', x)\n\ndf_train[\"question_text\"] = df_train[\"question_text\"].apply(remove_numbers)\ndf_test[\"question_text\"] = df_test[\"question_text\"].apply(remove_numbers)\n\ndf_train.head() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Removing Contractions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_abbreviation(data):\n        data = re.sub(r\"he's\", \"he is\", data)\n        data = re.sub(r\"there's\", \"there is\", data)\n        data = re.sub(r\"We're\", \"We are\", data)\n        data = re.sub(r\"That's\", \"That is\", data)\n        data = re.sub(r\"won't\", \"will not\", data)\n        data = re.sub(r\"they're\", \"they are\", data)\n        data = re.sub(r\"Can't\", \"Cannot\", data)\n        data = re.sub(r\"wasn't\", \"was not\", data)\n        data = re.sub(r\"don\\x89Ûªt\", \"do not\", data)\n        data= re.sub(r\"aren't\", \"are not\", data)\n        data = re.sub(r\"isn't\", \"is not\", data)\n        data = re.sub(r\"What's\", \"What is\", data)\n        data = re.sub(r\"haven't\", \"have not\", data)\n        data = re.sub(r\"hasn't\", \"has not\", data)\n        data = re.sub(r\"There's\", \"There is\", data)\n        data = re.sub(r\"He's\", \"He is\", data)\n        data = re.sub(r\"It's\", \"It is\", data)\n        data = re.sub(r\"You're\", \"You are\", data)\n        data = re.sub(r\"I'M\", \"I am\", data)\n        data = re.sub(r\"shouldn't\", \"should not\", data)\n        data = re.sub(r\"wouldn't\", \"would not\", data)\n        data = re.sub(r\"i'm\", \"I am\", data)\n        data = re.sub(r\"I\\x89Ûªm\", \"I am\", data)\n        data = re.sub(r\"I'm\", \"I am\", data)\n        data = re.sub(r\"Isn't\", \"is not\", data)\n        data = re.sub(r\"Here's\", \"Here is\", data)\n        data = re.sub(r\"you've\", \"you have\", data)\n        data = re.sub(r\"you\\x89Ûªve\", \"you have\", data)\n        data = re.sub(r\"we're\", \"we are\", data)\n        data = re.sub(r\"what's\", \"what is\", data)\n        data = re.sub(r\"couldn't\", \"could not\", data)\n        data = re.sub(r\"we've\", \"we have\", data)\n        data = re.sub(r\"it\\x89Ûªs\", \"it is\", data)\n        data = re.sub(r\"doesn\\x89Ûªt\", \"does not\", data)\n        data = re.sub(r\"It\\x89Ûªs\", \"It is\", data)\n        data = re.sub(r\"Here\\x89Ûªs\", \"Here is\", data)\n        data = re.sub(r\"who's\", \"who is\", data)\n        data = re.sub(r\"I\\x89Ûªve\", \"I have\", data)\n        data = re.sub(r\"y'all\", \"you all\", data)\n        data = re.sub(r\"can\\x89Ûªt\", \"cannot\", data)\n        data = re.sub(r\"would've\", \"would have\", data)\n        data = re.sub(r\"it'll\", \"it will\", data)\n        data = re.sub(r\"we'll\", \"we will\", data)\n        data = re.sub(r\"wouldn\\x89Ûªt\", \"would not\", data)\n        data = re.sub(r\"We've\", \"We have\", data)\n        data = re.sub(r\"he'll\", \"he will\", data)\n        data = re.sub(r\"Y'all\", \"You all\", data)\n        data = re.sub(r\"Weren't\", \"Were not\", data)\n        data = re.sub(r\"Didn't\", \"Did not\", data)\n        data = re.sub(r\"they'll\", \"they will\", data)\n        data = re.sub(r\"they'd\", \"they would\", data)\n        data = re.sub(r\"DON'T\", \"DO NOT\", data)\n        data = re.sub(r\"That\\x89Ûªs\", \"That is\", data)\n        data = re.sub(r\"they've\", \"they have\", data)\n        data = re.sub(r\"i'd\", \"I would\", data)\n        data = re.sub(r\"should've\", \"should have\", data)\n        data = re.sub(r\"You\\x89Ûªre\", \"You are\", data)\n        data = re.sub(r\"where's\", \"where is\", data)\n        data = re.sub(r\"Don\\x89Ûªt\", \"Do not\", data)\n        data = re.sub(r\"we'd\", \"we would\", data)\n        data = re.sub(r\"i'll\", \"I will\", data)\n        data = re.sub(r\"weren't\", \"were not\", data)\n        data = re.sub(r\"They're\", \"They are\", data)\n        data = re.sub(r\"Can\\x89Ûªt\", \"Cannot\", data)\n        data = re.sub(r\"you\\x89Ûªll\", \"you will\", data)\n        data = re.sub(r\"I\\x89Ûªd\", \"I would\", data)\n        data = re.sub(r\"let's\", \"let us\", data)\n        data = re.sub(r\"it's\", \"it is\", data)\n        data = re.sub(r\"can't\", \"cannot\", data)\n        data = re.sub(r\"dont\", \"do not\", data)\n        data = re.sub(r\"don't\", \"do not\", data)\n        data = re.sub(r\"you're\", \"you are\", data)\n        data = re.sub(r\"i've\", \"I have\", data)\n        data = re.sub(r\"that's\", \"that is\", data)\n        data = re.sub(r\"i'll\", \"I will\", data)\n        data = re.sub(r\"doesn't\", \"does not\",data)\n        data = re.sub(r\"i'd\", \"I would\", data)\n        data = re.sub(r\"didn't\", \"did not\", data)\n        data = re.sub(r\"ain't\", \"am not\", data)\n        data = re.sub(r\"you'll\", \"you will\", data)\n        data = re.sub(r\"I've\", \"I have\", data)\n        data = re.sub(r\"Don't\", \"do not\", data)\n        data = re.sub(r\"I'll\", \"I will\", data)\n        data = re.sub(r\"I'd\", \"I would\", data)\n        data = re.sub(r\"Let's\", \"Let us\", data)\n        data = re.sub(r\"you'd\", \"You would\", data)\n        data = re.sub(r\"It's\", \"It is\", data)\n        data = re.sub(r\"Ain't\", \"am not\", data)\n        data = re.sub(r\"Haven't\", \"Have not\", data)\n        data = re.sub(r\"Could've\", \"Could have\", data)\n        data = re.sub(r\"youve\", \"you have\", data)  \n        data = re.sub(r\"donå«t\", \"do not\", data)\n\n        return data\n    \ndf_train[\"question_text\"] = df_train[\"question_text\"].apply(remove_abbreviation)\ndf_test[\"question_text\"] = df_test[\"question_text\"].apply(remove_abbreviation)\ndf_train.to_csv(\"df_train.csv\", index = False)\ndf_test.to_csv(\"df_test.csv\", index = False)\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# remove stopwords\nStopwords are the English words which does not add much meaning to a sentence. They can safely be ignored without sacrificing the meaning of the sentence. For example, the words like the, he, have etc."},{"metadata":{"trusted":true},"cell_type":"code","source":"# remove stopwords function \ndef remove_stopwords(x):\n    stopword_list = set(stopwords.words('english'))\n    tokenizer = ToktokTokenizer()\n    tokens = tokenizer.tokenize(x)\n    tokens = [token.strip() for token in tokens]\n    filtered_tokens = [token for token in tokens if token not in stopword_list]\n    filtered_text = ' '.join(filtered_tokens)    \n    return filtered_text\n\ndf_train[\"question_text\"] = df_train[\"question_text\"].apply(remove_stopwords)\ndf_test[\"question_text\"] = df_test[\"question_text\"].apply(remove_stopwords)\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#  Lemmatization\n\nthe verbes with the form to be as 'to walk' may appear as 'walk', 'walked', 'walks', 'walking'. The base form, 'walk', that one might look up in a dictionary, is called the lemma for the word."},{"metadata":{"trusted":true},"cell_type":"code","source":"#Lemmatization\ndef lemma_text(x):\n    wordnet_lemmatizer = WordNetLemmatizer()\n    tokenizer = ToktokTokenizer()\n    tokens = tokenizer.tokenize(x)\n    tokens = [token.strip() for token in tokens]\n    tokens = [wordnet_lemmatizer.lemmatize(token) for token in tokens]\n    return ' '.join(tokens)\n\ndf_train[\"question_text\"] = df_train[\"question_text\"].apply(lemma_text)\ndf_test[\"question_text\"] = df_test[\"question_text\"].apply(lemma_text)\n\n\n\ndf_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import zipfile\nimport sys, os, re, csv, codecs, numpy as np, pandas as pd\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras import callbacks\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# **Split the dataset into train, validation**\n* 80% for trainig data\n* 20% for validation (test) data"},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train, val_df = train_test_split(df_train, test_size=0.2, random_state=42)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\n###   maxlen : maximum length of all sequences and shorter sequences are padded with zeros\nmaxlen = 100 # max number of words in a question to use\n\n\n## fill up the missing values\ntrain_X =  df_train[\"question_text\"].values\nval_X = val_df[\"question_text\"].values\ntest_X = df_test[\"question_text\"].values\n\n\n##Standard keras preprocessing, to turn each comment into a list of word indexes of equal length (with truncation or padding as needed).\n## Tokenize the sentences\n### Turns questions (strings) into lists of integer indices \n### Use only unique word (not repeat word)\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X)+list(val_X))\ntrain_X_tokens = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X_tokens = tokenizer.texts_to_sequences(test_X)\n\n\n## Pad the sentences \n###   Make all sequences in same length.\ntrain_X_pad = pad_sequences(train_X_tokens, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X_pad = pad_sequences(test_X_tokens, maxlen=maxlen)\n\n## Get the target values\ntrain_y = df_train['target'].values\nval_y = val_df['target'].values\n\ntrain_X_pad\n\n# test_X_pad\n# train_X_pad","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_y","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Embedding"},{"metadata":{"trusted":true},"cell_type":"code","source":"with zipfile.ZipFile(\"../input/quora-insincere-questions-classification/embeddings.zip\",\"r\") as z:\n    z.extractall(\".\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### Glove : Is a model for distributed word representation where the distance between words \n### is related to semantic similarity\n#Read the glove word vectors (space delimited strings) into a dictionary from word->vector.\nEMBEDDING_FILE ='./glove.840B.300d/glove.840B.300d.txt' \n\n\ndef get_coefs(word,*arr):\n    return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nemb_mean,emb_std","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_index = tokenizer.word_index\n\nnb_words = min(max_features, len(word_index))\n\n\n### embedding_matrix : matrix which contain each line a vector with a corresponding word\n###Use these vectors to create our embedding matrix, with random initialization for words that aren't in GloVe. We'll use the same mean and stdev of embeddings the GloVe has when generating the random init.\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#inp = Input(shape=(maxlen,))\n#x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n#x = Bidirectional(LSTM(50, return_sequences=True, dropout=0.1, recurrent_dropout=0.1))(x)\n#x = GlobalMaxPool1D()(x)\n#x = Dense(50, activation=\"relu\")(x)\n#x = Dropout(0.1)(x)\n#x = Dense(1, activation=\"sigmoid\")(x)\n#model = Model(inputs=inp, outputs=x)\n#model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n#print(model.summary())\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# * Create a model"},{"metadata":{"trusted":true},"cell_type":"code","source":"### A model is a function which gives us a near correct information (output) given that we've provided input data\nfrom keras.models import Sequential\nmodel = Sequential()\nmodel.add(Embedding(max_features, embed_size , weights=[embedding_matrix], trainable=False))\nmodel.add(LSTM(32\n              ))\nmodel.add(Dense(32,activation='relu'))\nmodel.add(Dropout(0.3))\nmodel.add(Dense(1, activation=\"sigmoid\"\n         ))\n\n### Loss function is a measure of how well our model’s outputs match the targets which we are trying to minimize.\n### Regression => mean squared error  (mse = sum(y - y(predict))²), mean absolute error (MAE)\n### Classfication => categorical cross entropy (multi-class data), binary cross entropy (for binary prediction)\n### In our poblem  we use  binary crossentropy \n\n### Optimizer : algorithm that helps our loss function reach its convergence point with minimum\n### the optimization algorithm varies the parameters of the model (weights and bias ).\n### This operation is repeated until we find the values of the parameters, for which the Loss function is optimal.\n\nmodel.compile(loss='binary_crossentropy', optimizer='adam' ,metrics=['accuracy'])\nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"### This callback allows us to interrupt training as soon as the model start overfitting (The validation loss starts to increase)\n### After 5 epoch with no improvement (val_loss(i) to val_loss(i+5) Increase) the training will be stopped\nes = callbacks.EarlyStopping( patience=5 )\n\n### This callback lets us continually save the model during training\n### Save_best_only=True, save_weights_only=True => keep the best model seen during training.\nmc = callbacks.ModelCheckpoint('./w.h5', save_best_only=True, save_weights_only=True)\n\n### Lancer l'apprentissage du  modèle pour 10000 époques (10000 itérations sur tous les échantillons dans le x_train et  y_train), en 'batch_size' de 512 échantillons.\n### validation_data : Helps us to prevent overfitting\nmodel.fit(train_X_pad, train_y, batch_size=512, epochs=20000, callbacks=[es, mc ], validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#y_test = model.predict([test_X_pad], batch_size=1024, verbose=1)\n#y_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sample_submission = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')\n#sample_submission['prediction'] = y_test\n#sample_submission.to_csv('submission.csv', index=False)\n#sample_submission","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##And finally, get predictions for the test set and prepare a submission CSV:\nout = model.predict(test_X_pad,batch_size=256)\nout_df = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\nout_pred = (out>0.35).astype(int)\nout_df['prediction'] = out_pred\nout_df.to_csv(\"submission.csv\", index=False)\nout_df","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}