{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"# Import library\n\nfrom __future__ import absolute_import, division, print_function, unicode_literals\n\ntry:\n  %tensorflow_version 2.x\nexcept Exception:\n  pass\n   \nimport tensorflow as tf\nfrom tensorflow import keras\n\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport tensorflow_datasets as tfds\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\n\n\n\nimport os\nprint(tf.__version__)\n ","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:11:23.415085Z","iopub.execute_input":"2021-05-23T14:11:23.415547Z","iopub.status.idle":"2021-05-23T14:11:30.309229Z","shell.execute_reply.started":"2021-05-23T14:11:23.415429Z","shell.execute_reply":"2021-05-23T14:11:30.307451Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Import dataset\n\nimport pandas as pd\ntrain_data = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest_data = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:11:34.026019Z","iopub.execute_input":"2021-05-23T14:11:34.026402Z","iopub.status.idle":"2021-05-23T14:11:40.719544Z","shell.execute_reply.started":"2021-05-23T14:11:34.026367Z","shell.execute_reply":"2021-05-23T14:11:40.718827Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean lower\n\ndef clean_lower(df):\n    df[\"question_text\"] = df[\"question_text\"].apply(lambda x: x.lower())\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:11:41.730233Z","iopub.execute_input":"2021-05-23T14:11:41.730896Z","iopub.status.idle":"2021-05-23T14:11:41.736303Z","shell.execute_reply.started":"2021-05-23T14:11:41.730843Z","shell.execute_reply":"2021-05-23T14:11:41.735383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean punctuation\n\npuncts = [\n    ',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&',\n    '/', '[', ']', '%', '=', '#', '*', '+', '\\\\', '•', '~', '@', '£',\n    '·', '_', '{', '}', '©', '^', '®', '`', '→', '°', '€', '™', '›',\n    '♥', '←', '×', '§', '″', '′', 'Â', '█', 'à', '…', '“', '★', '”',\n    '–', '●', 'â', '►', '−', '¢', '¬', '░', '¶', '↑', '±',  '▾',\n    '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '⊕', '▼',\n    '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲',\n    'è', '¸', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',\n    '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',\n    '¹', '≤', '‡', '₹', '´'\n]\n\ndef _clean_puncts(x, puncts):\n    x = str(x)\n    # added space around puncts after replace\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_puncts(df, puncts):\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_puncts(x, puncts))\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:11:45.410286Z","iopub.execute_input":"2021-05-23T14:11:45.410643Z","iopub.status.idle":"2021-05-23T14:11:45.422247Z","shell.execute_reply.started":"2021-05-23T14:11:45.410613Z","shell.execute_reply":"2021-05-23T14:11:45.421099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean numbers\n\ndef _clean_numbers(x):\n if bool(re.search(r'\\d', x)):\n     x = re.sub('[0–9]{5,}', '#####', x)\n     x = re.sub('[0–9]{4}', '####', x)\n     x = re.sub('[0–9]{3}', '###', x)\n     x = re.sub('[0–9]{2}', '##', x)\n return x\n\ndef clean_numbers(df, puncts):\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_numbers(x))\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:11:48.069855Z","iopub.execute_input":"2021-05-23T14:11:48.070380Z","iopub.status.idle":"2021-05-23T14:11:48.076801Z","shell.execute_reply.started":"2021-05-23T14:11:48.070347Z","shell.execute_reply":"2021-05-23T14:11:48.075793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Correcting mispelled words\n\nmispell_dict = {\n    \"colour\": \"color\", \n    \"centre\": \"center\", \n    \"favourite\": \"favorite\", \n    \"travelling\": \"traveling\", \n    \"counselling\": \"counseling\", \n    \"theatre\": \"theater\", \n    \"cancelled\": \"canceled\", \n    \"labour\": \"labor\", \n    \"organisation\": \"organization\", \n    \"wwii\": \"world war 2\", \n    \"citicise\": \"criticize\", \n    \"youtu \": \"youtube\", \n    \"Qoura\": \"Quora\", \n    \"sallary\": \"salary\", \n    \"Whta\": \"What\", \n    \"narcisist\": \"narcissist\", \n    \"howdo\": \"how do\", \n    \"whatare\": \"what are\", \n    \"howcan\": \"how can\", \n    \"howmuch\": \"how much\", \n    \"howmany\": \"how many\", \n    \"whydo\": \"why do\", \n    \"doI\": \"do I\", \n    \"theBest\": \"the best\", \n    \"howdoes\": \"how does\", \n    \"mastrubation\": \"masturbation\", \n    \"mastrubate\": \"masturbate\", \n    \"mastrubating\": \"masturbating\", \n    \"pennis\": \"penis\", \n    \"Etherium\": \"bitcoin\", \n    \"narcissit\": \"narcissist\", \n    \"bigdata\": \"big data\", \n    \"2k17\": \"2017\", \n    \"2k18\": \"2018\", \n    \"qouta\": \"quota\", \n    \"exboyfriend\": \"ex boyfriend\", \n    \"airhostess\": \"air hostess\", \n    \"whst\": \"what\", \n    \"watsapp\": \"whatsapp\", \n    \"demonitisation\": \"demonetization\", \n    \"demonitization\": \"demonetization\", \n    \"demonetisation\": \"demonetization\", \n    \"electroneum\": \"bitcoin\",\n    \"nanodegree\": \"degree\",\n    \"hotstar\": \"star\",\n    \"dream11\": \"dream\",\n    \"ftre\": \"fire\",\n    \"tensorflow\": \"framework\",\n    \"unocoin\": \"bitcoin\",\n    \"lnmiit\": \"limit\", \n    \"unacademy\": \"academy\",\n    \"altcoin\": \"bitcoin\",\n    \"altcoins\": \"bitcoin\", \n    \"litecoin\": \"bitcoin\",\n    \"coinbase\": \"bitcoin\",\n    \"cryptocurency\": \"cryptocurrency\",\n    \"simpliv\": \"simple\",\n    \"quoras\": \"quora\",\n    \"schizoids\": \"psychopath\",\n    \"remainers\": \"remainder\",\n    \"twinflame\": \"soulmate\",\n    \"quorans\": \"quora\",\n    \"brexit\": \"demonetized\",\n    \"iiest\": \"institute\",\n    \"dceu\": \"comics\",\n    \"pessat\": \"exam\", \n    \"uceed\": \"college\",\n    \"bhakts\": \"devotee\",\n    \"boruto\": \"anime\",\n    \"cryptocoin\": \"bitcoin\",\n    \"blockchains\": \"blockchain\",\n    \"fiancee\": \"fiance\",\n    \"redmi\": \"smartphone\",\n    \"oneplus\": \"smartphone\",\n    \"qoura\": \"quora\",\n    \"deepmind\": \"framework\",\n    \"ryzen\": \"cpu\",\n    \"whattsapp\": \"whatsapp\",\n    \"undertale\": \"adventure\",\n    \"zenfone\": \"smartphone\",\n    \"cryptocurencies\": \"cryptocurrencies\",\n    \"koinex\": \"bitcoin\",\n    \"zebpay\": \"bitcoin\",\n    \"binance\": \"bitcoin\",\n    \"whtsapp\": \"whatsapp\",\n    \"reactjs\": \"framework\",\n    \"bittrex\": \"bitcoin\",\n    \"bitconnect\": \"bitcoin\",\n    \"bitfinex\": \"bitcoin\",\n    \"yourquote\": \"your quote\",\n    \"whyis\": \"why is\",\n    \"jiophone\": \"smartphone\",\n    \"dogecoin\": \"bitcoin\",\n    \"onecoin\": \"bitcoin\", \n    \"poloniex\": \"bitcoin\",\n    \"7700k\": \"cpu\",\n    \"angular2\": \"framework\",\n    \"segwit2x\": \"bitcoin\",\n    \"hashflare\": \"bitcoin\", \n    \"940mx\": \"gpu\",\n    \"openai\": \"framework\",\n    \"hashflare\": \"bitcoin\",\n    \"1050ti\": \"gpu\",\n    \"nearbuy\": \"near buy\",\n    \"freebitco\": \"bitcoin\",\n    \"antminer\": \"bitcoin\",\n    \"filecoin\": \"bitcoin\", \n    \"whatapp\": \"whatsapp\",\n    \"empowr\": \"empower\",\n    \"1080ti\": \"gpu\",\n    \"crytocurrency\": \"cryptocurrency\",\n    \"8700k\": \"cpu\",\n    \"whatsaap\": \"whatsapp\",\n    \"g4560\": \"cpu\",\n    \"payymoney\": \"pay money\",\n    \"fuckboys\": \"fuck boys\",\n    \"intenship\": \"internship\",\n    \"zcash\": \"bitcoin\",\n    \"demonatisation\": \"demonetization\",\n    \"narcicist\": \"narcissist\",\n    \"mastuburation\": \"masturbation\",\n    \"trignometric\": \"trigonometric\",\n    \"cryptocurreny\": \"cryptocurrency\",\n    \"howdid\": \"how did\",\n    \"crytocurrencies\": \"cryptocurrencies\",\n    \"phycopath\": \"psychopath\",\n    \"bytecoin\": \"bitcoin\",\n    \"possesiveness\": \"possessiveness\",\n    \"scollege\": \"college\",\n    \"humanties\": \"humanities\",\n    \"altacoin\": \"bitcoin\",\n    \"demonitised\": \"demonetized\",\n    \"brasília\": \"brazilia\",\n    \"accolite\": \"accolyte\",\n    \"econimics\": \"economics\",\n    \"varrier\": \"warrier\",\n    \"quroa\": \"quora\",\n    \"statergy\": \"strategy\",\n    \"langague\": \"language\",\n    \"splatoon\": \"game\",\n    \"7600k\": \"cpu\",\n    \"gate2018\": \"gate 2018\",\n    \"in2018\": \"in 2018\",\n    \"narcassist\": \"narcissist\",\n    \"jiocoin\": \"bitcoin\",\n    \"hnlu\": \"hulu\",\n    \"7300hq\": \"cpu\",\n    \"weatern\": \"western\",\n    \"interledger\": \"blockchain\",\n    \"deplation\": \"deflation\", \n    \"cryptocurrencies\": \"cryptocurrency\", \n    \"bitcoin\": \"blockchain cryptocurrency\"\n}\n\ndef _correct_mispell(x, compiled_re, replace):\n    return compiled_re.sub(replace, x)\n\ndef correct_mispell(df, mispell_dict):\n    mispelled_word = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    def replace(match):\n        return mispell_dict[match.group(0)]\n    df['question_text'] = df['question_text'].apply(\n        lambda x: _correct_mispell(x, mispelled_word, replace)\n    )\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:11:53.117701Z","iopub.execute_input":"2021-05-23T14:11:53.118292Z","iopub.status.idle":"2021-05-23T14:11:53.141316Z","shell.execute_reply.started":"2021-05-23T14:11:53.118241Z","shell.execute_reply":"2021-05-23T14:11:53.140120Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Removing Contraction\n\nabbreviations = {\n    \"ain't\": \"is not\",\n    \"aren't\": \"are not\",\n    \"can't\": \"cannot\",\n    \"'cause\": \"because\",\n    \"could've\": \"could have\",\n    \"couldn't\": \"could not\",\n    \"didn't\": \"did not\",\n    \"doesn't\": \"does not\",\n    \"don't\": \"do not\",\n    \"hadn't\": \"had not\",\n    \"hasn't\": \"has not\",\n    \"haven't\": \"have not\",\n    \"he'd\": \"he would\",\n    \"he'll\": \"he will\",\n    \"he's\": \"he is\",\n    \"how'd\": \"how did\",\n    \"how'd'y\": \"how do you\",\n    \"how'll\": \"how will\",\n    \"how's\": \"how is\",\n    \"I'd\": \"I would\",\n    \"I'd've\": \"I would have\",\n    \"I'll\": \"I will\",\n    \"I'll've\": \"I will have\",\n    \"I'm\": \"I am\",\n    \"I've\": \"I have\",\n    \"i'd\": \"i would\",\n    \"i'd've\": \"i would have\",\n    \"i'll\": \"i will\",\n    \"i'll've\": \"i will have\",\n    \"i'm\": \"i am\",\n    \"i've\": \"i have\",\n    \"isn't\": \"is not\",\n    \"it'd\": \"it would\",\n    \"it'd've\": \"it would have\",\n    \"it'll\": \"it will\",\n    \"it'll've\": \"it will have\",\n    \"it's\": \"it is\",\n    \"let's\": \"let us\",\n    \"ma'am\": \"madam\",\n    \"mayn't\": \"may not\",\n    \"might've\": \"might have\",\n    \"mightn't\": \"might not\",\n    \"mightn't've\": \"might not have\",\n    \"must've\": \"must have\",\n    \"mustn't\": \"must not\",\n    \"mustn't've\": \"must not have\",\n    \"needn't\": \"need not\",\n    \"needn't've\": \"need not have\",\n    \"o'clock\": \"of the clock\",\n    \"oughtn't\": \"ought not\",\n    \"oughtn't've\": \"ought not have\",\n    \"shan't\": \"shall not\",\n    \"sha'n't\": \"shall not\",\n    \"shan't've\": \"shall not have\",\n    \"she'd\": \"she would\",\n    \"she'd've\": \"she would have\",\n    \"she'll\": \"she will\",\n    \"she'll've\": \"she will have\",\n    \"she's\": \"she is\",\n    \"should've\": \"should have\",\n    \"shouldn't\": \"should not\",\n    \"shouldn't've\": \"should not have\",\n    \"so've\": \"so have\",\n    \"so's\": \"so as\",\n    \"this's\": \"this is\",\n    \"that'd\": \"that would\",\n    \"that'd've\": \"that would have\",\n    \"that's\": \"that is\",\n    \"there'd\": \"there would\",\n    \"there'd've\": \"there would have\",\n    \"there's\": \"there is\",\n    \"here's\": \"here is\",\n    \"they'd\": \"they would\",\n     \"they'd've\": \"they would have\",\n    \"they'll\": \"they will\",\n    \"they'll've\": \"they will have\",\n    \"they're\": \"they are\",\n    \"they've\": \"they have\",\n    \"to've\": \"to have\",\n    \"wasn't\": \"was not\",\n    \"we'd\": \"we would\",\n    \"we'd've\": \"we would have\",\n    \"we'll\": \"we will\",\n    \"we'll've\": \"we will have\",\n    \"we're\": \"we are\",\n    \"we've\": \"we have\",\n    \"weren't\": \"were not\",\n    \"what'll\": \"what will\",\n    \"what'll've\": \"what will have\",\n    \"what're\": \"what are\",\n    \"what's\": \"what is\",\n    \"what've\": \"what have\",\n    \"when's\": \"when is\",\n    \"when've\": \"when have\",\n    \"where'd\": \"where did\",\n    \"where's\": \"where is\",\n    \"where've\": \"where have\",\n     \"who'll\": \"who will\",\n    \"who'll've\": \"who will have\",\n    \"who's\": \"who is\",\n    \"who've\": \"who have\",\n    \"why's\": \"why is\",\n    \"why've\": \"why have\",\n    \"will've\": \"will have\",\n    \"won't\": \"will not\",\n    \"won't've\": \"will not have\",\n    \"would've\": \"would have\",\n    \"wouldn't\": \"would not\",\n    \"wouldn't've\": \"would not have\",\n    \"y'all\": \"you all\",\n    \"y'all'd\": \"you all would\",\n    \"y'all'd've\": \"you all would have\",\n    \"y'all're\": \"you all are\",\n    \"y'all've\": \"you all have\",\n    \"you'd\": \"you would\",\n    \"you'd've\": \"you would have\",\n    \"you'll\": \"you will\",\n    \"you'll've\": \"you will have\",\n    \"you're\": \"you are\",\n    \"you've\": \"you have\",\n    \"who'd\": \"who would\",\n    \"who're\": \"who are\",\n    \"'re\": \" are\",\n    \"tryin'\": \"trying\",\n    \"doesn'\": \"does not\",\n    'howdo': 'how do',\n    'whatare': 'what are',\n    'howcan': 'how can',\n    'howmuch': 'how much',\n    'howmany': 'how many',\n    'whydo': 'why do',\n    'doI': 'do I',\n    'theBest': 'the best',\n    'howdoes': 'how does',\n}\n\ndef _clean_abreviation(x, compiled_re, replace):\n    return compiled_re.sub(replace, x)\n\ndef clean_abbreviation(df, abbreviations):\n    compiled_abbreviation = re.compile('(%s)' % '|'.join(abbreviations.keys()))\n    def replace(match):\n        return abbreviations[match.group(0)]\n    df['question_text'] = df[\"question_text\"].apply(\n        lambda x: _clean_abreviation(x, compiled_abbreviation, replace)\n    )\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:12:00.607499Z","iopub.execute_input":"2021-05-23T14:12:00.607872Z","iopub.status.idle":"2021-05-23T14:12:00.628137Z","shell.execute_reply.started":"2021-05-23T14:12:00.607841Z","shell.execute_reply":"2021-05-23T14:12:00.626977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Remove stopword\n\nimport nltk\nstopword_list = nltk.corpus.stopwords.words(\"english\")\n\ndef _remove_stopwords(text, is_lower_case=True):\n tokenizer = ToktokTokenizer()\n tokens = tokenizer.tokenize(text)\n tokens = [token.strip() for token in tokens]\n if is_lower_case:\n     filtered_tokens = [token for token in tokens if token not in stopword_list]\n else:\n     filtered_tokens = [token for token in tokens if token.lower() not in stopword_list]\n filtered_text = \" \".join(filtered_tokens)\n return filtered_text\n\ndef remove_stopwords(df):\n    df['question_text'] = df['question_text'].apply(lambda x: _remove_stopwords(x))\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:12:05.458127Z","iopub.execute_input":"2021-05-23T14:12:05.458501Z","iopub.status.idle":"2021-05-23T14:12:06.934560Z","shell.execute_reply.started":"2021-05-23T14:12:05.458464Z","shell.execute_reply":"2021-05-23T14:12:06.933507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Clean math\n\ndef _clean_math(x, compiled_re):\n    return compiled_re.sub(' <math> ', x)\n\ndef clean_math(df):\n    math_puncts = 'θπα÷⁴≠β²¾∫≥⇒¬∠＝∑Φ√½¼'\n    math_puncts_long = [r'\\\\frac', r'\\[math\\]', r'\\[/math\\]', r'\\\\lim']\n    compiled_math = re.compile('(%s)' % '|'.join(math_puncts))\n    compiled_math_long = re.compile('(%s)' % '|'.join(math_puncts_long))\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math_long))\n    df['question_text'] = df['question_text'].apply(lambda x: _clean_math(x, compiled_math))\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:12:10.814014Z","iopub.execute_input":"2021-05-23T14:12:10.814407Z","iopub.status.idle":"2021-05-23T14:12:10.822216Z","shell.execute_reply.started":"2021-05-23T14:12:10.814371Z","shell.execute_reply":"2021-05-23T14:12:10.820942Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Stemming\n\nfrom nltk.stem import SnowballStemmer\nfrom nltk.tokenize.toktok import ToktokTokenizer\n\ndef _stem_text(text):\n tokenizer = ToktokTokenizer()\n stemmer = SnowballStemmer(\"english\")\n tokens = tokenizer.tokenize(text)\n tokens = [token.strip() for token in tokens]\n tokens = [stemmer.stem(token) for token in tokens]\n return \" \".join(tokens)\n\ndef stem_text(df):\n    df['question_text'] = df['question_text'].apply(lambda x: _stem_text(x))\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:12:13.818228Z","iopub.execute_input":"2021-05-23T14:12:13.818602Z","iopub.status.idle":"2021-05-23T14:12:13.825924Z","shell.execute_reply.started":"2021-05-23T14:12:13.818567Z","shell.execute_reply":"2021-05-23T14:12:13.824844Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lemmatization\n\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize.toktok import ToktokTokenizer\n\nwordnet_lemmatizer = WordNetLemmatizer()\ndef _lemma_text(text):\n tokenizer = ToktokTokenizer()\n tokens = tokenizer.tokenize(text)\n tokens = [token.strip() for token in tokens]\n tokens = [wordnet_lemmatizer.lemmatize(token) for token in tokens]\n return \" \".join(tokens)\n\ndef lema_text(df):\n    df['question_text'] = df['question_text'].apply(lambda x: _lemma_text(x))\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:12:17.214650Z","iopub.execute_input":"2021-05-23T14:12:17.215126Z","iopub.status.idle":"2021-05-23T14:12:17.223305Z","shell.execute_reply.started":"2021-05-23T14:12:17.215092Z","shell.execute_reply":"2021-05-23T14:12:17.222262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Function to clean dataset\n\ndef clean(df):\n    df = clean_lower(df)\n    df = clean_puncts(df, puncts)\n    df = clean_numbers(df, puncts)\n    df = correct_mispell(df, mispell_dict)\n    df = clean_abbreviation(df, abbreviations)\n    df = remove_stopwords(df)\n    df = clean_math(df)\n    df = stem_text(df)\n    df = lema_text(df)\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:12:19.802481Z","iopub.execute_input":"2021-05-23T14:12:19.802940Z","iopub.status.idle":"2021-05-23T14:12:19.809855Z","shell.execute_reply.started":"2021-05-23T14:12:19.802906Z","shell.execute_reply":"2021-05-23T14:12:19.808549Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Function to run cleaning process\n\nfrom multiprocessing import Pool\nimport re\n\nnum_cores = 2\ndef df_parallelize_run(df, func, num_cores=2):\n    df_split = np.array_split(df, num_cores)\n    pool = Pool(num_cores)\n    df = pd.concat(pool.map(func, df_split))\n    pool.close()\n    pool.join()\n    return df","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:12:22.194065Z","iopub.execute_input":"2021-05-23T14:12:22.194456Z","iopub.status.idle":"2021-05-23T14:12:22.202045Z","shell.execute_reply.started":"2021-05-23T14:12:22.194423Z","shell.execute_reply":"2021-05-23T14:12:22.200812Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Cleaning dataset\n\ntrain_data = df_parallelize_run(train_data, clean)\ntest_data = df_parallelize_run(test_data, clean)\nprint(\"Train shape : \", train_data.shape)\nprint(\"Test shape : \", test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:12:24.715254Z","iopub.execute_input":"2021-05-23T14:12:24.715845Z","iopub.status.idle":"2021-05-23T14:19:30.418631Z","shell.execute_reply.started":"2021-05-23T14:12:24.715803Z","shell.execute_reply":"2021-05-23T14:19:30.417344Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Divide training dataset\n\nX = train_data[\"question_text\"].values\ny = train_data[\"target\"].values","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:20:08.556143Z","iopub.execute_input":"2021-05-23T14:20:08.556550Z","iopub.status.idle":"2021-05-23T14:20:08.562477Z","shell.execute_reply.started":"2021-05-23T14:20:08.556510Z","shell.execute_reply":"2021-05-23T14:20:08.561670Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tokenize dataset\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom sklearn.model_selection import train_test_split\nfrom keras.utils.np_utils import to_categorical\nfrom sklearn.preprocessing import LabelEncoder\n\ntrunc_type='post'\npadding_type='post'\n\ndef tokenize_word(data):\n  tokenizer = Tokenizer()\n  tokenizer.fit_on_texts(data)\n  tokenized_data = tokenizer.texts_to_sequences(data)\n  tokenized_data = pad_sequences(tokenized_data, dtype='int64', padding='post')  \n  return tokenized_data, tokenizer","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:20:11.251185Z","iopub.execute_input":"2021-05-23T14:20:11.251769Z","iopub.status.idle":"2021-05-23T14:20:11.328659Z","shell.execute_reply.started":"2021-05-23T14:20:11.251701Z","shell.execute_reply":"2021-05-23T14:20:11.327907Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tokenize training dataset\nX, tokenizer = tokenize_word(X)\nprint(tokenizer)\n\n# Split training dataset into training and testing samples\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=37)\n\n# Get vocabulary size\nvocab_size = len(tokenizer.word_index) + 2","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:20:17.654333Z","iopub.execute_input":"2021-05-23T14:20:17.654983Z","iopub.status.idle":"2021-05-23T14:21:07.608409Z","shell.execute_reply.started":"2021-05-23T14:20:17.654926Z","shell.execute_reply":"2021-05-23T14:21:07.607471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Build up model\n\nmodel = tf.keras.Sequential()\nmodel.add(tf.keras.layers.Embedding(vocab_size, 64))\nmodel.add(tf.keras.layers.Bidirectional(tf.keras.layers.LSTM(64,)))\n\n# One or more dense layers.\n# Edit the list in the `for` line to experiment with layer sizes.\nfor units in [64, 64]:\n  model.add(tf.keras.layers.Dense(units, activation='relu'))\n\n# Output layer. The first argument is the number of labels.\nmodel.add(tf.keras.layers.Dense(2, activation='softmax'))\nprint(model.summary()) ","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:21:29.094491Z","iopub.execute_input":"2021-05-23T14:21:29.095025Z","iopub.status.idle":"2021-05-23T14:21:29.822764Z","shell.execute_reply.started":"2021-05-23T14:21:29.094982Z","shell.execute_reply":"2021-05-23T14:21:29.821546Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Add optimizer\n\nmodel.compile(optimizer='adam',\n              loss='sparse_categorical_crossentropy',\n              metrics=['accuracy'])","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:21:34.990177Z","iopub.execute_input":"2021-05-23T14:21:34.990560Z","iopub.status.idle":"2021-05-23T14:21:35.008092Z","shell.execute_reply.started":"2021-05-23T14:21:34.990527Z","shell.execute_reply":"2021-05-23T14:21:35.006957Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Training model\n\nhistory= model.fit(X_train, y_train,validation_split=0.1, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T14:21:37.330119Z","iopub.execute_input":"2021-05-23T14:21:37.330509Z","iopub.status.idle":"2021-05-23T16:39:22.030758Z","shell.execute_reply.started":"2021-05-23T14:21:37.330478Z","shell.execute_reply":"2021-05-23T16:39:22.029856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Save model\nmodel.save('../output/kaggle/working/quora_prediction.h5')\n\n# Load model\nreconstructed_model = tf.keras.models.load_model('../output/kaggle/working/quora_prediction.h5')","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:39:41.102218Z","iopub.execute_input":"2021-05-23T16:39:41.102630Z","iopub.status.idle":"2021-05-23T16:39:42.669912Z","shell.execute_reply.started":"2021-05-23T16:39:41.102593Z","shell.execute_reply":"2021-05-23T16:39:42.668930Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get label\n\ndef get_label(word_index):\n  if prediction[word_index][0] > prediction[word_index][1]:\n    return 0\n  else:\n    return 1 ","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:39:44.625075Z","iopub.execute_input":"2021-05-23T16:39:44.625446Z","iopub.status.idle":"2021-05-23T16:39:44.631237Z","shell.execute_reply.started":"2021-05-23T16:39:44.625416Z","shell.execute_reply":"2021-05-23T16:39:44.630361Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction on testing sample\n\nprediction = reconstructed_model.predict(X_test)\ny_pred = [get_label(i) for i in range(len(X_test))]","metadata":{"execution":{"iopub.status.busy":"2021-05-23T11:51:58.936961Z","iopub.execute_input":"2021-05-23T11:51:58.937331Z","iopub.status.idle":"2021-05-23T11:54:00.940055Z","shell.execute_reply.started":"2021-05-23T11:51:58.937301Z","shell.execute_reply":"2021-05-23T11:54:00.939098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualize confusion matrix\n\nfrom sklearn.metrics import confusion_matrix\ncnf_matrix = confusion_matrix(y_pred, y_test)\n\nimport matplotlib.pyplot as plt\nimport itertools\ndef plot_confusion_matrix(cm, classes,\n                          normalize=False,\n                          title='Confusion matrix',\n                          cmap=plt.cm.Blues):\n    if normalize:\n        cm = cm.astype('float') / cm.sum(axis=1, keepdims = True)\n\n    plt.imshow(cm, interpolation='nearest', cmap=cmap)\n    plt.title(title)\n    plt.colorbar()\n    tick_marks = np.arange(len(classes))\n    plt.xticks(tick_marks, classes, rotation=45)\n    plt.yticks(tick_marks, classes)\n\n    fmt = '.2f' if normalize else 'd'\n    thresh = cm.max() / 2.\n    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n        plt.text(j, i, format(cm[i, j], fmt),\n                 horizontalalignment=\"center\",\n                 color=\"white\" if cm[i, j] > thresh else \"black\")\n\n    plt.tight_layout()\n    plt.ylabel('True label')\n    plt.xlabel('Predicted label')\n\n# Plot non-normalized confusion matrix\nclass_names = [0, 1]\nplt.figure()\nplot_confusion_matrix(cnf_matrix, classes=class_names,\n                      title='Confusion matrix, without normalization')\nplt.show()\n\n# Plot normalized confusion matrix\nplt.figure()\nplot_confusion_matrix(cnf_matrix, classes=class_names, normalize=True,\n                      title='Normalized confusion matrix')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-23T11:54:16.742002Z","iopub.execute_input":"2021-05-23T11:54:16.74266Z","iopub.status.idle":"2021-05-23T11:54:17.54213Z","shell.execute_reply.started":"2021-05-23T11:54:16.742605Z","shell.execute_reply":"2021-05-23T11:54:17.5412Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Caculate critical scores\n\nfrom sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score\n\n# Caculate accuracy score \naccuracy = accuracy_score(y_pred, y_test)\nprint('Accuracy: %f' % accuracy)\n\n# Caculate precision score\nprecision = precision_score(y_pred, y_test)\nprint('Precision: %f' % precision)\n\n# Caculate recall score\nrecall = recall_score(y_pred, y_test)\nprint('Recall: %f' % recall)\n\n# Caculate F1 score\nf1 = f1_score(y_pred, y_test)\nprint('F1 score: %f' % f1)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T11:54:21.616797Z","iopub.execute_input":"2021-05-23T11:54:21.617173Z","iopub.status.idle":"2021-05-23T11:54:22.242749Z","shell.execute_reply.started":"2021-05-23T11:54:21.617137Z","shell.execute_reply":"2021-05-23T11:54:22.241631Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Tokenize testing dataset\ntokenized_test_data, tokenizer = tokenize_word(test_data[\"question_text\"])","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:39:59.910484Z","iopub.execute_input":"2021-05-23T16:39:59.911059Z","iopub.status.idle":"2021-05-23T16:40:15.416070Z","shell.execute_reply.started":"2021-05-23T16:39:59.911007Z","shell.execute_reply":"2021-05-23T16:40:15.414691Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(tokenized_test_data)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:40:27.225773Z","iopub.execute_input":"2021-05-23T16:40:27.226205Z","iopub.status.idle":"2021-05-23T16:40:27.232500Z","shell.execute_reply.started":"2021-05-23T16:40:27.226169Z","shell.execute_reply":"2021-05-23T16:40:27.231318Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prediction on testing dataset\n\nprediction = reconstructed_model.predict(tokenized_test_data)\ntest_data_pred = [get_label(i) for i in range(len(test_data))]","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:40:29.621079Z","iopub.execute_input":"2021-05-23T16:40:29.621468Z","iopub.status.idle":"2021-05-23T16:48:15.806079Z","shell.execute_reply.started":"2021-05-23T16:40:29.621435Z","shell.execute_reply":"2021-05-23T16:48:15.804953Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Submit result\n\nresults = test_data[['qid']]\nresults['prediction'] = test_data_pred\nresults.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2021-05-23T16:50:03.014073Z","iopub.execute_input":"2021-05-23T16:50:03.014602Z","iopub.status.idle":"2021-05-23T16:50:04.038250Z","shell.execute_reply.started":"2021-05-23T16:50:03.014570Z","shell.execute_reply":"2021-05-23T16:50:04.037065Z"},"trusted":true},"execution_count":null,"outputs":[]}]}