{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# 1. Khảo sát dữ liệu","metadata":{}},{"cell_type":"code","source":"import pandas as pd \nimport seaborn as sns\nimport re\nimport gc\nimport os\nimport numpy as np\nimport operator\nfrom wordcloud import WordCloud, STOPWORDS\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\npd_ctx = pd.option_context('display.max_colwidth', 100)\n\nimport nltk\nfrom nltk.stem import PorterStemmer, SnowballStemmer, WordNetLemmatizer\n\nfrom gensim.models import KeyedVectors\n\nimport tensorflow as tf\n\nfrom sklearn.model_selection import train_test_split\nimport sklearn.metrics as metrics\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D,GRU\nfrom tensorflow.keras.layers import Bidirectional, GlobalMaxPool1D, SpatialDropout1D, GlobalMaxPooling1D, Concatenate\nfrom tensorflow.keras.models import Model,load_model\nfrom tensorflow.keras.utils import plot_model\nfrom tensorflow.keras.models import Sequential\nfrom tensorflow.keras.callbacks import ModelCheckpoint,EarlyStopping\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:28:54.797491Z","iopub.execute_input":"2022-01-06T17:28:54.798315Z","iopub.status.idle":"2022-01-06T17:28:54.808747Z","shell.execute_reply.started":"2022-01-06T17:28:54.798274Z","shell.execute_reply":"2022-01-06T17:28:54.807873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Xác định các tập **train**, **test** và khởi tạo một tập chứa dữ liệu của cả 2 để về sau check độ bao phủ của vocab với tập dữ liệu đã sinh","metadata":{}},{"cell_type":"code","source":"df_train = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv')\ndf_test = pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv')\nquora_data = df_train['question_text'].append(df_test['question_text'])","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:10.227153Z","iopub.execute_input":"2022-01-06T17:27:10.227406Z","iopub.status.idle":"2022-01-06T17:27:16.739519Z","shell.execute_reply.started":"2022-01-06T17:27:10.227379Z","shell.execute_reply":"2022-01-06T17:27:16.738449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"a. Khảo sát dữ liệu trong tập train**","metadata":{}},{"cell_type":"code","source":"print(\"\\033[1mTrain set info\\033[0m\")\nprint(df_train.info())","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:16.740646Z","iopub.execute_input":"2022-01-06T17:27:16.740910Z","iopub.status.idle":"2022-01-06T17:27:17.058209Z","shell.execute_reply.started":"2022-01-06T17:27:16.740880Z","shell.execute_reply":"2022-01-06T17:27:17.057140Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ở trong bộ dữ liệu có 3 trường, và khả năng dữ liệu train nằm ở trường **question_text**  \nVậy ta sẽ thử khảo sát dữ liệu trong trường **question_text**","metadata":{}},{"cell_type":"code","source":"# Kiểm tra các trường dữ liệu của câu hỏi được đánh sincere\nprint(\"\\033[1mSincere Questions: \\033[0m\")\ndisplay(df_train[df_train['target']==0].head())\n# Kiểm tra các trường dữ liệu của câu hỏi được đánh sincere\nprint(\"\\033[1mInsincere Questions: \\033[0m\")\ndisplay(df_train[df_train['target']==1].head())","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:17.060527Z","iopub.execute_input":"2022-01-06T17:27:17.060760Z","iopub.status.idle":"2022-01-06T17:27:17.213898Z","shell.execute_reply.started":"2022-01-06T17:27:17.060733Z","shell.execute_reply":"2022-01-06T17:27:17.212672Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.target.value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:17.215349Z","iopub.execute_input":"2022-01-06T17:27:17.216004Z","iopub.status.idle":"2022-01-06T17:27:17.232056Z","shell.execute_reply.started":"2022-01-06T17:27:17.215956Z","shell.execute_reply":"2022-01-06T17:27:17.230885Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pos_len = len(df_train[df_train['target'] == 1])\nneg_len = len(df_train[df_train['target'] == 0])\ntotal = len(df_train)\nprint(\"\\033[1mTotal = \\033[0m\", total)\nprint(\"\\033[1mSincere questions:\\033[0m {neg} ({percent: .2f}% )\".format(neg = neg_len, percent = neg_len / total * 100))\nprint(\"\\033[1mInsincere questions:\\033[0m {pos} ({percent: .2f}% )\".format(pos = pos_len, percent = pos_len / total * 100))\nfig = plt.figure()\nax = fig.add_axes([0,0,1,1])\nax.bar(['sincere', 'insincere'], df_train.target.value_counts())\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:17.233744Z","iopub.execute_input":"2022-01-06T17:27:17.234091Z","iopub.status.idle":"2022-01-06T17:27:17.549171Z","shell.execute_reply.started":"2022-01-06T17:27:17.234049Z","shell.execute_reply":"2022-01-06T17:27:17.548247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"b. Khảo sát dữ liệu trong tập test**","metadata":{}},{"cell_type":"code","source":"df_test.info()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:17.550367Z","iopub.execute_input":"2022-01-06T17:27:17.550609Z","iopub.status.idle":"2022-01-06T17:27:17.646245Z","shell.execute_reply.started":"2022-01-06T17:27:17.550581Z","shell.execute_reply":"2022-01-06T17:27:17.645555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Shuffle tập train để kiểm tra những giá trị ngẫu nhiên\ntrain = df_train.sample(frac=1).reset_index(drop=True)\ndisplay(train.sample(n=10, random_state=344))","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:17.647555Z","iopub.execute_input":"2022-01-06T17:27:17.647848Z","iopub.status.idle":"2022-01-06T17:27:18.556387Z","shell.execute_reply.started":"2022-01-06T17:27:17.647772Z","shell.execute_reply":"2022-01-06T17:27:18.555500Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 2. Tiền xử lý dữ liệu","metadata":{}},{"cell_type":"code","source":"### do việc lemming words yêu cầu sử dụng thư viện wordnet để lookup các từ có trong đó\n# nltk.download('wordnet')\n# nltk.download('punkt')\ndef clean_tag(x):\n  if '[math]' in x:\n    x = re.sub('\\[math\\].*?math\\]', 'MATH EQUATION', x) #replacing with [MATH EQUATION]\n    \n  if 'http' in x or 'www' in x:\n    x = re.sub('(?:(?:https?|ftp):\\/\\/)?[\\w/\\-?=%.]+\\.[\\w/\\-?=%.]+', 'URL', x) #replacing with [url]\n  return x","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:18.557740Z","iopub.execute_input":"2022-01-06T17:27:18.557979Z","iopub.status.idle":"2022-01-06T17:27:18.563464Z","shell.execute_reply.started":"2022-01-06T17:27:18.557953Z","shell.execute_reply":"2022-01-06T17:27:18.562558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', \n        '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', \n        '█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', \n        '—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞', \n        '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━', \n        '⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○', \n        '◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐', \n        '├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\\x97', '↺', \n        '∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊', \n        '┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´', \n        '↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\\x93', '≧', '］', \n        '\\x94', '∀', '♛', '\\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％', \n        '╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭', \n        '┊', '╯', '☾', '➔', '∴', '\\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐', \n        '▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\\x91', '➚', '◡', '╰', '\\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒', \n        '☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜', \n        '‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢', \n        '≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥', \n        '⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ', \n        '✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦', \n        '◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ', \n        '⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋', \n        '､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞', \n        '◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ', \n        '✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱', \n        '☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹', \n        '⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚', \n        '☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛', \n        '⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙', \n        'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘', \n        '˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛', \n        '☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑', \n        '✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ', \n        '⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘', \n        '⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊', \n        '⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷', \n        '◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝', \n        '⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢', \n        '≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']\n\ndef clean_punct(x):\n  x = str(x)\n  for punct in puncts:\n    if punct in x:\n      x = x.replace(punct, ' ')\n    return x","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:18.564886Z","iopub.execute_input":"2022-01-06T17:27:18.565099Z","iopub.status.idle":"2022-01-06T17:27:18.608933Z","shell.execute_reply.started":"2022-01-06T17:27:18.565075Z","shell.execute_reply":"2022-01-06T17:27:18.607936Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'bitcoin', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', \n                'electroneum':'bitcoin','nanodegree':'degree','hotstar':'star','dream11':'dream','ftre':'fire','tensorflow':'framework','unocoin':'bitcoin',\n                'lnmiit':'limit','unacademy':'academy','altcoin':'bitcoin','altcoins':'bitcoin','litecoin':'bitcoin','coinbase':'bitcoin','cryptocurency':'cryptocurrency',\n                'simpliv':'simple','quoras':'quora','schizoids':'psychopath','remainers':'remainder','twinflame':'soulmate','quorans':'quora','brexit':'demonetized',\n                'iiest':'institute','dceu':'comics','pessat':'exam','uceed':'college','bhakts':'devotee','boruto':'anime',\n                'cryptocoin':'bitcoin','blockchains':'blockchain','fiancee':'fiance','redmi':'smartphone','oneplus':'smartphone','qoura':'quora','deepmind':'framework','ryzen':'cpu','whattsapp':'whatsapp',\n                'undertale':'adventure','zenfone':'smartphone','cryptocurencies':'cryptocurrencies','koinex':'bitcoin','zebpay':'bitcoin','binance':'bitcoin','whtsapp':'whatsapp',\n                'reactjs':'framework','bittrex':'bitcoin','bitconnect':'bitcoin','bitfinex':'bitcoin','yourquote':'your quote','whyis':'why is','jiophone':'smartphone',\n                'dogecoin':'bitcoin','onecoin':'bitcoin','poloniex':'bitcoin','7700k':'cpu','angular2':'framework','segwit2x':'bitcoin','hashflare':'bitcoin','940mx':'gpu',\n                'openai':'framework','hashflare':'bitcoin','1050ti':'gpu','nearbuy':'near buy','freebitco':'bitcoin','antminer':'bitcoin','filecoin':'bitcoin','whatapp':'whatsapp',\n                'empowr':'empower','1080ti':'gpu','crytocurrency':'cryptocurrency','8700k':'cpu','whatsaap':'whatsapp','g4560':'cpu','payymoney':'pay money',\n                'fuckboys':'fuck boys','intenship':'internship','zcash':'bitcoin','demonatisation':'demonetization','narcicist':'narcissist','mastuburation':'masturbation',\n                'trignometric':'trigonometric','cryptocurreny':'cryptocurrency','howdid':'how did','crytocurrencies':'cryptocurrencies','phycopath':'psychopath',\n                'bytecoin':'bitcoin','possesiveness':'possessiveness','scollege':'college','humanties':'humanities','altacoin':'bitcoin','demonitised':'demonetized',\n                'brasília':'brazilia','accolite':'accolyte','econimics':'economics','varrier':'warrier','quroa':'quora','statergy':'strategy','langague':'language',\n                'splatoon':'game','7600k':'cpu','gate2018':'gate 2018','in2018':'in 2018','narcassist':'narcissist','jiocoin':'bitcoin','hnlu':'hulu','7300hq':'cpu',\n                'weatern':'western','interledger':'blockchain','deplation':'deflation', 'cryptocurrencies':'cryptocurrency', 'bitcoin':'blockchain cryptocurrency',}\n\ndef correct_mispell(x):\n  words = x.split()\n  for i in range(0, len(words)):\n    if mispell_dict.get(words[i]) is not None:\n      words[i] = mispell_dict.get(words[i])\n    elif mispell_dict.get(words[i].lower()) is not None:\n      words[i] = mispell_dict.get(words[i].lower())\n        \n  words = \" \".join(words)\n  return words","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:18.610947Z","iopub.execute_input":"2022-01-06T17:27:18.611271Z","iopub.status.idle":"2022-01-06T17:27:18.632771Z","shell.execute_reply.started":"2022-01-06T17:27:18.611229Z","shell.execute_reply":"2022-01-06T17:27:18.631982Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def remove_stopwords(x):\n  x = [word for word in x.split() if word not in STOPWORDS]\n  x = ' '.join(x)\n  return x","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:18.634038Z","iopub.execute_input":"2022-01-06T17:27:18.634282Z","iopub.status.idle":"2022-01-06T17:27:18.647506Z","shell.execute_reply.started":"2022-01-06T17:27:18.634255Z","shell.execute_reply":"2022-01-06T17:27:18.646904Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contraction_mapping = {\n \"I'm\": 'I am',\n \"I'm'a\": 'I am about to',\n \"I'm'o\": 'I am going to',\n \"I've\": 'I have',\n \"I'll\": 'I will',\n \"I'll've\": 'I will have',\n \"I'd\": 'I would',\n \"I'd've\": 'I would have',\n \"i'm\": 'i am',\n \"i'm'a\": 'i am about to',\n \"i'm'o\": 'i am going to',\n \"i've\": 'i have',\n \"i'll\": 'i will',\n \"i'll've\": 'i will have',\n \"i'd\": 'i would',\n \"i'd've\": 'i would have',\n 'Whatcha': 'What are you',\n 'whatcha': 'what are you',\n \"amn't\": 'am not',\n \"ain't\": 'are not',\n \"aren't\": 'are not',\n \"'cause\": 'because',\n \"can't\": 'can not',\n \"can't've\": 'can not have',\n \"could've\": 'could have',\n \"couldn't\": 'could not',\n \"couldn't've\": 'could not have',\n \"daren't\": 'dare not',\n \"daresn't\": 'dare not',\n \"dasn't\": 'dare not',\n \"didn't\": 'did not',\n 'didn’t': 'did not',\n \"don't\": 'do not',\n 'don’t': 'do not',\n \"doesn't\": 'does not',\n \"e'er\": 'ever',\n \"everyone's\": 'everyone is',\n 'finna': 'fixing to',\n 'gimme': 'give me',\n \"gon't\": 'go not',\n 'gonna': 'going to',\n 'gotta': 'got to',\n \"hadn't\": 'had not',\n \"hadn't've\": 'had not have',\n \"hasn't\": 'has not',\n \"haven't\": 'have not',\n \"he've\": 'he have',\n \"he's\": 'he is',\n \"he'll\": 'he will',\n \"he'll've\": 'he will have',\n \"he'd\": 'he would',\n \"he'd've\": 'he would have',\n \"here's\": 'here is',\n \"how're\": 'how are',\n \"how'd\": 'how did',\n \"how'd'y\": 'how do you',\n \"how's\": 'how is',\n \"how'll\": 'how will',\n \"isn't\": 'is not',\n \"it's\": 'it is',\n \"'tis\": 'it is',\n \"'twas\": 'it was',\n \"it'll\": 'it will',\n \"it'll've\": 'it will have',\n \"it'd\": 'it would',\n \"it'd've\": 'it would have',\n 'kinda': 'kind of',\n \"let's\": 'let us',\n 'luv': 'love',\n \"ma'am\": 'madam',\n \"may've\": 'may have',\n \"mayn't\": 'may not',\n \"might've\": 'might have',\n \"mightn't\": 'might not',\n \"mightn't've\": 'might not have',\n \"must've\": 'must have',\n \"mustn't\": 'must not',\n \"mustn't've\": 'must not have',\n \"needn't\": 'need not',\n \"needn't've\": 'need not have',\n \"ne'er\": 'never',\n \"o'\": 'of',\n \"o'clock\": 'of the clock',\n \"ol'\": 'old',\n \"oughtn't\": 'ought not',\n \"oughtn't've\": 'ought not have',\n \"o'er\": 'over',\n \"shan't\": 'shall not',\n \"sha'n't\": 'shall not',\n \"shalln't\": 'shall not',\n \"shan't've\": 'shall not have',\n \"she's\": 'she is',\n \"she'll\": 'she will',\n \"she'd\": 'she would',\n \"she'd've\": 'she would have',\n \"should've\": 'should have',\n \"shouldn't\": 'should not',\n \"shouldn't've\": 'should not have',\n \"so've\": 'so have',\n \"so's\": 'so is',\n \"somebody's\": 'somebody is',\n \"someone's\": 'someone is',\n \"something's\": 'something is',\n 'sux': 'sucks',\n \"that're\": 'that are',\n \"that's\": 'that is',\n \"that'll\": 'that will',\n \"that'd\": 'that would',\n \"that'd've\": 'that would have',\n 'em': 'them',\n \"there're\": 'there are',\n \"there's\": 'there is',\n \"there'll\": 'there will',\n \"there'd\": 'there would',\n \"there'd've\": 'there would have',\n \"these're\": 'these are',\n \"they're\": 'they are',\n \"they've\": 'they have',\n \"they'll\": 'they will',\n \"they'll've\": 'they will have',\n \"they'd\": 'they would',\n \"they'd've\": 'they would have',\n \"this's\": 'this is',\n \"those're\": 'those are',\n \"to've\": 'to have',\n 'wanna': 'want to',\n \"wasn't\": 'was not',\n \"we're\": 'we are',\n \"we've\": 'we have',\n \"we'll\": 'we will',\n \"we'll've\": 'we will have',\n \"we'd\": 'we would',\n \"we'd've\": 'we would have',\n \"weren't\": 'were not',\n \"what're\": 'what are',\n \"what'd\": 'what did',\n \"what've\": 'what have',\n \"what's\": 'what is',\n \"what'll\": 'what will',\n \"what'll've\": 'what will have',\n \"when've\": 'when have',\n \"when's\": 'when is',\n \"where're\": 'where are',\n \"where'd\": 'where did',\n \"where've\": 'where have',\n \"where's\": 'where is',\n \"which's\": 'which is',\n \"who're\": 'who are',\n \"who've\": 'who have',\n \"who's\": 'who is',\n \"who'll\": 'who will',\n \"who'll've\": 'who will have',\n \"who'd\": 'who would',\n \"who'd've\": 'who would have',\n \"why're\": 'why are',\n \"why'd\": 'why did',\n \"why've\": 'why have',\n \"why's\": 'why is',\n \"will've\": 'will have',\n \"won't\": 'will not',\n \"won't've\": 'will not have',\n \"would've\": 'would have',\n \"wouldn't\": 'would not',\n \"wouldn't've\": 'would not have',\n \"y'all\": 'you all',\n \"y'all're\": 'you all are',\n \"y'all've\": 'you all have',\n \"y'all'd\": 'you all would',\n \"y'all'd've\": 'you all would have',\n \"you're\": 'you are',\n \"you've\": 'you have',\n \"you'll've\": 'you shall have',\n \"you'll\": 'you will',\n \"you'd\": 'you would',\n \"you'd've\": 'you would have',\n 'jan.': 'january',\n 'feb.': 'february',\n 'mar.': 'march',\n 'apr.': 'april',\n 'jun.': 'june',\n 'jul.': 'july',\n 'aug.': 'august',\n 'sep.': 'september',\n 'oct.': 'october',\n 'nov.': 'november',\n 'dec.': 'december',\n 'I’m': 'I am',\n 'I’m’a': 'I am about to',\n 'I’m’o': 'I am going to',\n 'I’ve': 'I have',\n 'I’ll': 'I will',\n 'I’ll’ve': 'I will have',\n 'I’d': 'I would',\n 'I’d’ve': 'I would have',\n 'i’m': 'i am',\n 'i’m’a': 'i am about to',\n 'i’m’o': 'i am going to',\n 'i’ve': 'i have',\n 'i’ll': 'i will',\n 'i’ll’ve': 'i will have',\n 'i’d': 'i would',\n 'i’d’ve': 'i would have',\n 'amn’t': 'am not',\n 'ain’t': 'are not',\n 'aren’t': 'are not',\n '’cause': 'because',\n 'can’t': 'can not',\n 'can’t’ve': 'can not have',\n 'could’ve': 'could have',\n 'couldn’t': 'could not',\n 'couldn’t’ve': 'could not have',\n 'daren’t': 'dare not',\n 'daresn’t': 'dare not',\n 'dasn’t': 'dare not',\n 'doesn’t': 'does not',\n 'e’er': 'ever',\n 'everyone’s': 'everyone is',\n 'gon’t': 'go not',\n 'hadn’t': 'had not',\n 'hadn’t’ve': 'had not have',\n 'hasn’t': 'has not',\n 'haven’t': 'have not',\n 'he’ve': 'he have',\n 'he’s': 'he is',\n 'he’ll': 'he will',\n 'he’ll’ve': 'he will have',\n 'he’d': 'he would',\n 'he’d’ve': 'he would have',\n 'here’s': 'here is',\n 'how’re': 'how are',\n 'how’d': 'how did',\n 'how’d’y': 'how do you',\n 'how’s': 'how is',\n 'how’ll': 'how will',\n 'isn’t': 'is not',\n 'it’s': 'it is',\n '’tis': 'it is',\n '’twas': 'it was',\n 'it’ll': 'it will',\n 'it’ll’ve': 'it will have',\n 'it’d': 'it would',\n 'it’d’ve': 'it would have',\n 'let’s': 'let us',\n 'ma’am': 'madam',\n 'may’ve': 'may have',\n 'mayn’t': 'may not',\n 'might’ve': 'might have',\n 'mightn’t': 'might not',\n 'mightn’t’ve': 'might not have',\n 'must’ve': 'must have',\n 'mustn’t': 'must not',\n 'mustn’t’ve': 'must not have',\n 'needn’t': 'need not',\n 'needn’t’ve': 'need not have',\n 'ne’er': 'never',\n 'o’': 'of',\n 'o’clock': 'of the clock',\n 'ol’': 'old',\n 'oughtn’t': 'ought not',\n 'oughtn’t’ve': 'ought not have',\n 'o’er': 'over',\n 'shan’t': 'shall not',\n 'sha’n’t': 'shall not',\n 'shalln’t': 'shall not',\n 'shan’t’ve': 'shall not have',\n 'she’s': 'she is',\n 'she’ll': 'she will',\n 'she’d': 'she would',\n 'she’d’ve': 'she would have',\n 'should’ve': 'should have',\n 'shouldn’t': 'should not',\n 'shouldn’t’ve': 'should not have',\n 'so’ve': 'so have',\n 'so’s': 'so is',\n 'somebody’s': 'somebody is',\n 'someone’s': 'someone is',\n 'something’s': 'something is',\n 'that’re': 'that are',\n 'that’s': 'that is',\n 'that’ll': 'that will',\n 'that’d': 'that would',\n 'that’d’ve': 'that would have',\n 'there’re': 'there are',\n 'there’s': 'there is',\n 'there’ll': 'there will',\n 'there’d': 'there would',\n 'there’d’ve': 'there would have',\n 'these’re': 'these are',\n 'they’re': 'they are',\n 'they’ve': 'they have',\n 'they’ll': 'they will',\n 'they’ll’ve': 'they will have',\n 'they’d': 'they would',\n 'they’d’ve': 'they would have',\n 'this’s': 'this is',\n 'those’re': 'those are',\n 'to’ve': 'to have',\n 'wasn’t': 'was not',\n 'we’re': 'we are',\n 'we’ve': 'we have',\n 'we’ll': 'we will',\n 'we’ll’ve': 'we will have',\n 'we’d': 'we would',\n 'we’d’ve': 'we would have',\n 'weren’t': 'were not',\n 'what’re': 'what are',\n 'what’d': 'what did',\n 'what’ve': 'what have',\n 'what’s': 'what is',\n 'what’ll': 'what will',\n 'what’ll’ve': 'what will have',\n 'when’ve': 'when have',\n 'when’s': 'when is',\n 'where’re': 'where are',\n 'where’d': 'where did',\n 'where’ve': 'where have',\n 'where’s': 'where is',\n 'which’s': 'which is',\n 'who’re': 'who are',\n 'who’ve': 'who have',\n 'who’s': 'who is',\n 'who’ll': 'who will',\n 'who’ll’ve': 'who will have',\n 'who’d': 'who would',\n 'who’d’ve': 'who would have',\n 'why’re': 'why are',\n 'why’d': 'why did',\n 'why’ve': 'why have',\n 'why’s': 'why is',\n 'will’ve': 'will have',\n 'won’t': 'will not',\n 'won’t’ve': 'will not have',\n 'would’ve': 'would have',\n 'wouldn’t': 'would not',\n 'wouldn’t’ve': 'would not have',\n 'y’all': 'you all',\n 'y’all’re': 'you all are',\n 'y’all’ve': 'you all have',\n 'y’all’d': 'you all would',\n 'y’all’d’ve': 'you all would have',\n 'you’re': 'you are',\n 'you’ve': 'you have',\n 'you’ll’ve': 'you shall have',\n 'you’ll': 'you will',\n 'you’d': 'you would',\n 'you’d’ve': 'you would have'\n}\n\ndef clean_contractions(text):\n#     text = text.lower()\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    \n    text = ' '.join([contraction_mapping[t] if t in contraction_mapping else t for t in text.split(\" \")])\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:18.651167Z","iopub.execute_input":"2022-01-06T17:27:18.651406Z","iopub.status.idle":"2022-01-06T17:27:18.697504Z","shell.execute_reply.started":"2022-01-06T17:27:18.651381Z","shell.execute_reply":"2022-01-06T17:27:18.696800Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# lemmatizer = WordNetLemmatizer()\n# def lemma_text(x):\n#   x = x.split()\n#   x = [lemmatizer.lemmatize(word) for word in x]\n#   x = ' '.join(x)\n#   return x","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:18.699112Z","iopub.execute_input":"2022-01-06T17:27:18.699676Z","iopub.status.idle":"2022-01-06T17:27:18.715185Z","shell.execute_reply.started":"2022-01-06T17:27:18.699634Z","shell.execute_reply":"2022-01-06T17:27:18.714313Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def data_cleaning(x):\n  x = clean_tag(x)\n  x = clean_punct(x)\n  x = correct_mispell(x)\n  x = remove_stopwords(x)\n  x = clean_contractions(x)\n#   x = lemma_text(x)\n  return x","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:18.716822Z","iopub.execute_input":"2022-01-06T17:27:18.717159Z","iopub.status.idle":"2022-01-06T17:27:18.727598Z","shell.execute_reply.started":"2022-01-06T17:27:18.717121Z","shell.execute_reply":"2022-01-06T17:27:18.726965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Preprocess(doc):\n    corpus=[]\n    for text in tqdm(doc):\n        text=clean_contractions(text)\n        text=correct_mispell(text)\n        text=re.sub(r'[^a-z0-9A-Z]',\" \",text)\n        text=re.sub(r'[0-9]{1}',\"#\",text)\n        text=re.sub(r'[0-9]{2}','##',text)\n        text=re.sub(r'[0-9]{3}','###',text)\n        text=re.sub(r'[0-9]{4}','####',text)\n        text=re.sub(r'[0-9]{5,}','#####',text)\n        corpus.append(text)\n    return corpus","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:18.729867Z","iopub.execute_input":"2022-01-06T17:27:18.730397Z","iopub.status.idle":"2022-01-06T17:27:18.740564Z","shell.execute_reply.started":"2022-01-06T17:27:18.730357Z","shell.execute_reply":"2022-01-06T17:27:18.739647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"b. Tiền xử lý các tập dữ liệu","metadata":{}},{"cell_type":"code","source":"tqdm.pandas(desc=\"progress-bar\")\ndf_test['question_text_cleaned'] = df_test['question_text'].progress_map(lambda x: data_cleaning(x))\ndf_train['question_text_cleaned'] = df_train['question_text'].progress_map(lambda x: data_cleaning(x))","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:18.741691Z","iopub.execute_input":"2022-01-06T17:27:18.741968Z","iopub.status.idle":"2022-01-06T17:27:28.797295Z","shell.execute_reply.started":"2022-01-06T17:27:18.741940Z","shell.execute_reply":"2022-01-06T17:27:28.794555Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_train.head(5)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.798342Z","iopub.status.idle":"2022-01-06T17:27:28.798704Z","shell.execute_reply.started":"2022-01-06T17:27:28.798510Z","shell.execute_reply":"2022-01-06T17:27:28.798532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 3. Build tập vocab","metadata":{}},{"cell_type":"code","source":"%%time\n### unzipping all the pretrained embeddings\n!unzip ../input/quora-insincere-questions-classification/embeddings.zip\n!du -h ./","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.800315Z","iopub.status.idle":"2022-01-06T17:27:28.800672Z","shell.execute_reply.started":"2022-01-06T17:27:28.800490Z","shell.execute_reply":"2022-01-06T17:27:28.800511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n\n### Loading tập Google News Pretrained Embeddings vào bộ nhớ\nfile_name=\"./GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\"\nmodel_embed=KeyedVectors.load_word2vec_format(file_name,binary=True)\n# model_embed = load_embed1('./glove.840B.300d/glove.840B.300d.txt')\n\n### Xây dựng tập từ vựng dựa trên dữ liệu của tập Google News\ndef vocab_build(corpus):\n    vocab={}\n    for text in tqdm(corpus):\n        for word in text.split():\n            try:\n                vocab[word]+=1\n            except KeyError:\n                vocab[word]=1\n    return vocab\n\n\n### Kiểm tra tập Vocabulary xem tập vocab đó bao phủ bao nhiêu phần trăm tập dữ liệu của mình\ndef check_voc(vocab,model):\n    embed_words=[]\n    out_vocab={}\n    total_words=0\n    total_text=0\n    for i in tqdm(vocab):\n        try:\n            vec=model[i]\n            embed_words.append(vec)\n            total_words+=vocab[i]\n        except KeyError:\n            out_vocab[i]=vocab[i]\n            total_text+=vocab[i]\n    print(\"The {:.2f}% of vocabularies have Covered of corpus\".format(100*len(embed_words)/len(vocab)))\n    print(\"The {:.2f}% of total text had coverded \".format((100*total_words/(total_words+total_text))))\n    return out_vocab","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.802284Z","iopub.status.idle":"2022-01-06T17:27:28.802632Z","shell.execute_reply.started":"2022-01-06T17:27:28.802450Z","shell.execute_reply":"2022-01-06T17:27:28.802471Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### xây tập vocab và kiểm tra độ phủ của tập vocab với dữ liệu đã được xử lý\ntotal_text=pd.concat([df_train.question_text_cleaned,df_test.question_text_cleaned])\nvocabulary=vocab_build(total_text)\noov=check_voc(vocabulary,model_embed) #oov: out of vocab","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.804107Z","iopub.status.idle":"2022-01-06T17:27:28.804659Z","shell.execute_reply.started":"2022-01-06T17:27:28.804454Z","shell.execute_reply":"2022-01-06T17:27:28.804473Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"df_test['question_text_cleaned_2'] = Preprocess(df_test['question_text'])\ndf_train['question_text_cleaned_2'] = Preprocess(df_train['question_text'])\ntotal_text_2=pd.concat([df_train.question_text_cleaned_2,df_test.question_text_cleaned_2])\nvocabulary2=vocab_build(total_text_2)\noov2=check_voc(vocabulary2, model_embed)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.805815Z","iopub.status.idle":"2022-01-06T17:27:28.806445Z","shell.execute_reply.started":"2022-01-06T17:27:28.806236Z","shell.execute_reply":"2022-01-06T17:27:28.806258Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sort_oov=dict(sorted(oov2.items(), key=operator.itemgetter(1),reverse=True))\ndict(list(sort_oov.items())[:50])","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.807502Z","iopub.status.idle":"2022-01-06T17:27:28.808124Z","shell.execute_reply.started":"2022-01-06T17:27:28.807915Z","shell.execute_reply":"2022-01-06T17:27:28.807945Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del oov, oov2,sort_oov,total_text,total_text_2\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.809201Z","iopub.status.idle":"2022-01-06T17:27:28.809577Z","shell.execute_reply.started":"2022-01-06T17:27:28.809372Z","shell.execute_reply":"2022-01-06T17:27:28.809393Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_word_index(vocab):\n    word_index=dict((w,i+1) for i,w in enumerate(vocab.keys()))\n    return word_index\ndef fit_one_hot(word_index,corpus):\n    sent=[]\n    for text in tqdm(corpus):\n        li=[]\n        for word in text.split():\n            try:\n                li.append(word_index[word])\n            except KeyError:\n                li.append(0)\n        sent.append(li)\n    return sent","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.810809Z","iopub.status.idle":"2022-01-06T17:27:28.811722Z","shell.execute_reply.started":"2022-01-06T17:27:28.811462Z","shell.execute_reply":"2022-01-06T17:27:28.811491Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train,val=train_test_split(df_train,test_size=0.2,stratify=df_train.target,random_state=123)\nvocab_size=len(vocabulary2)+1\nmax_len=40\n\nword_index=get_word_index(vocabulary2)\n### Chuẩn bị dữ liệu đã được xử lý\ntrain_text=train['question_text_cleaned_2']\nval_text=val['question_text_cleaned_2']\ntest_text=df_test['question_text_cleaned_2']\n\n### mã hóa câu trong tập train sang dạng onehot cho dễ xử lý\nencodes=fit_one_hot(word_index,train_text)\ntrain_padded=pad_sequences(encodes,maxlen=max_len,padding=\"post\")\n\n### mã hóa câu trong tập validation sang dạng onehot cho dễ xử lý\nencodes_=fit_one_hot(word_index,val_text)\nval_padded=pad_sequences(encodes_,maxlen=max_len,padding=\"post\")\n\n### mã hóa câu trong tập test sang dạng onehot cho dễ xử lý\nencodes__=fit_one_hot(word_index,test_text)\ntest_padded=pad_sequences(encodes__,maxlen=max_len,padding=\"post\")","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.812981Z","iopub.status.idle":"2022-01-06T17:27:28.813335Z","shell.execute_reply.started":"2022-01-06T17:27:28.813155Z","shell.execute_reply":"2022-01-06T17:27:28.813178Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count=0\nembedding_mat=np.zeros((vocab_size,300))\nfor word,i in tqdm(word_index.items()):\n    try:\n        vec=model_embed[word]\n        embedding_mat[i]=vec\n    except KeyError:\n        count+=1\n        continue\n\nprint(\"Number of Out of Vocabulary\",count)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.814753Z","iopub.status.idle":"2022-01-06T17:27:28.815306Z","shell.execute_reply.started":"2022-01-06T17:27:28.815119Z","shell.execute_reply":"2022-01-06T17:27:28.815139Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 4. Chuẩn bị Model","metadata":{}},{"cell_type":"code","source":"def get_model_origin(matrix):\n    inp = Input(shape=(max_len,))\n    x = Embedding(vocab_size,300,weights=[matrix],input_length=max_len,trainable=False)(inp)\n    x = Bidirectional(LSTM(128, return_sequences=True))(x)\n    x = Conv1D(64,3,activation=\"relu\")(x)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(128, activation=\"relu\")(x)\n    x = Dropout(0.2)(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.816216Z","iopub.status.idle":"2022-01-06T17:27:28.817032Z","shell.execute_reply.started":"2022-01-06T17:27:28.816836Z","shell.execute_reply":"2022-01-06T17:27:28.816860Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt=Adam(learning_rate=0.001)\nBATCH_SIZE = 1024\nbin_loss=tf.keras.losses.BinaryCrossentropy(from_logits=False,label_smoothing=0,name='binary_crossentropy')\n\n### Xác định điểm callback để giảm learning rate, và restore lại trọng số tốt nhất kề trước \nearly_stopping=tf.keras.callbacks.EarlyStopping(monitor=\"val_loss\",patience=3,mode=\"min\",restore_best_weights=True\n                                              )\n### Giảm learning rate khi model không được cải thiên (càng học càng ngu)\nreduce_lr=tf.keras.callbacks.ReduceLROnPlateau(monitor=\"val_loss\",factor=0.2,patience=2,verbose=1,mode=\"auto\")\n\nmy_callbacks=[early_stopping,reduce_lr]","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.818071Z","iopub.status.idle":"2022-01-06T17:27:28.818622Z","shell.execute_reply.started":"2022-01-06T17:27:28.818410Z","shell.execute_reply":"2022-01-06T17:27:28.818430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = None\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print('Use TPU')\nexcept ValueError:\n    if len(tf.config.list_physical_devices('GPU')) > 0:\n        strategy = tf.distribute.MirroredStrategy()\n        print('Use GPU')\n    else:\n        strategy = tf.distribute.get_strategy()\n        print('Use CPU')","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.819532Z","iopub.status.idle":"2022-01-06T17:27:28.819896Z","shell.execute_reply.started":"2022-01-06T17:27:28.819695Z","shell.execute_reply":"2022-01-06T17:27:28.819715Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 5. Huấn luyện và Dự đoán","metadata":{}},{"cell_type":"markdown","source":"a. Thử nghiệm model 1","metadata":{}},{"cell_type":"code","source":"with strategy.scope():\n    google_model = get_model_origin(embedding_mat)\n    google_model.compile(loss=bin_loss, optimizer=opt, metrics=['accuracy'])\nhistory=google_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.821674Z","iopub.status.idle":"2022-01-06T17:27:28.822148Z","shell.execute_reply.started":"2022-01-06T17:27:28.821953Z","shell.execute_reply":"2022-01-06T17:27:28.821974Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for loss\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 1)\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\n# plt.show()\n# summarize history for accuracy\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 2)\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.823144Z","iopub.status.idle":"2022-01-06T17:27:28.823453Z","shell.execute_reply.started":"2022-01-06T17:27:28.823294Z","shell.execute_reply":"2022-01-06T17:27:28.823309Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"google_y_pre=google_model.predict(val_padded, verbose=1)\nbest_score = 0\nbest_thresh = 0\nfor thresh in np.arange(0.1,0.5,0.01):\n    if(best_score < metrics.f1_score(val.target,(google_y_pre>thresh).astype(int))):\n        best_score = metrics.f1_score(val.target,(google_y_pre>thresh))\n        best_thresh = round(thresh, 2)\n    print(\"threshold {0:2.2f} f1 score:{1:2.3f}\".format(thresh,metrics.f1_score(val.target,(google_y_pre>thresh).astype(int))))\nprint(\"\\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\\033[0m\".format(best_score, best_thresh))","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.824505Z","iopub.status.idle":"2022-01-06T17:27:28.824845Z","shell.execute_reply.started":"2022-01-06T17:27:28.824650Z","shell.execute_reply":"2022-01-06T17:27:28.824665Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del model_embed, history, best_score, best_thresh, google_model, google_y_pre\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.826352Z","iopub.status.idle":"2022-01-06T17:27:28.826671Z","shell.execute_reply.started":"2022-01-06T17:27:28.826508Z","shell.execute_reply":"2022-01-06T17:27:28.826523Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"b. Thử nghiệm model 2","metadata":{}},{"cell_type":"code","source":"def get_model(matrix):\n    inp = Input(shape=(max_len,))\n    x = Embedding(vocab_size, 300, weights=[matrix], trainable=False)(inp)\n    x = SpatialDropout1D(0.3)(x)\n    x1 = Bidirectional(LSTM(256, return_sequences=True))(x)\n    x2 = Bidirectional(GRU(128, return_sequences=True))(x1)\n    max_pool1 = GlobalMaxPooling1D()(x1)\n    max_pool2 = GlobalMaxPooling1D()(x2)\n    conc = Concatenate()([max_pool1, max_pool2])\n    predictions = Dense(1, activation='sigmoid')(conc)\n    model = Model(inputs=inp, outputs=predictions)\n    adam = optimizers.Adam(lr=0.001)\n    model.compile(optimizer=adam, loss='binary_crossentropy', metrics=['accuracy'])\n    return model","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.827833Z","iopub.status.idle":"2022-01-06T17:27:28.828168Z","shell.execute_reply.started":"2022-01-06T17:27:28.827994Z","shell.execute_reply":"2022-01-06T17:27:28.828011Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = None\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print('Use TPU')\nexcept ValueError:\n    if len(tf.config.list_physical_devices('GPU')) > 0:\n        strategy = tf.distribute.MirroredStrategy()\n        print('Use GPU')\n    else:\n        strategy = tf.distribute.get_strategy()\n        print('Use CPU')","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.829878Z","iopub.status.idle":"2022-01-06T17:27:28.830370Z","shell.execute_reply.started":"2022-01-06T17:27:28.830105Z","shell.execute_reply":"2022-01-06T17:27:28.830131Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    google_model = get_model(embedding_mat)\n    google_model.compile(loss=bin_loss, optimizer=Adam(lr=0.001), metrics=['accuracy'])\nhistory=google_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.832075Z","iopub.status.idle":"2022-01-06T17:27:28.832555Z","shell.execute_reply.started":"2022-01-06T17:27:28.832296Z","shell.execute_reply":"2022-01-06T17:27:28.832320Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for loss\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 1)\nplt.plot(history.history['loss'])\nplt.plot(history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\n# plt.show()\n# summarize history for accuracy\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 2)\nplt.plot(history.history['accuracy'])\nplt.plot(history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.834064Z","iopub.status.idle":"2022-01-06T17:27:28.834544Z","shell.execute_reply.started":"2022-01-06T17:27:28.834287Z","shell.execute_reply":"2022-01-06T17:27:28.834312Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"google_y_pre=google_model.predict(val_padded, verbose=1)\nbest_score = 0\nbest_thresh = 0\nfor thresh in np.arange(0.1,0.5,0.01):\n    if(best_score < metrics.f1_score(val.target,(google_y_pre>thresh).astype(int))):\n        best_score = metrics.f1_score(val.target,(google_y_pre>thresh))\n        best_thresh = round(thresh, 2)\n    print(\"threshold {0:2.2f} f1 score:{1:2.3f}\".format(thresh,metrics.f1_score(val.target,(google_y_pre>thresh).astype(int))))\nprint(\"\\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\\033[0m\".format(best_score, best_thresh))","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.835988Z","iopub.status.idle":"2022-01-06T17:27:28.836501Z","shell.execute_reply.started":"2022-01-06T17:27:28.836229Z","shell.execute_reply":"2022-01-06T17:27:28.836256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold=best_thresh\ngoogle_y_test_pre=google_model.predict(test_padded, batch_size=BATCH_SIZE, verbose=1)\ngoogle_y_test_pre=(google_y_test_pre>thresh).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.839285Z","iopub.status.idle":"2022-01-06T17:27:28.839659Z","shell.execute_reply.started":"2022-01-06T17:27:28.839468Z","shell.execute_reply":"2022-01-06T17:27:28.839490Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del embedding_mat, history, best_score, best_thresh\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.841426Z","iopub.status.idle":"2022-01-06T17:27:28.841779Z","shell.execute_reply.started":"2022-01-06T17:27:28.841592Z","shell.execute_reply":"2022-01-06T17:27:28.841617Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# 6. So sánh, mở rộng","metadata":{}},{"cell_type":"code","source":"def load_embed(file):\n    def get_coefs(word,*arr):\n        return word, np.asarray(arr, dtype='float32')\n\n    if file == '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec':\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file) if len(o)>100)\n    else:\n        embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n\n    return embeddings_index","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.843374Z","iopub.status.idle":"2022-01-06T17:27:28.843981Z","shell.execute_reply.started":"2022-01-06T17:27:28.843738Z","shell.execute_reply":"2022-01-06T17:27:28.843767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!tree -h ./","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.845220Z","iopub.status.idle":"2022-01-06T17:27:28.845557Z","shell.execute_reply.started":"2022-01-06T17:27:28.845380Z","shell.execute_reply":"2022-01-06T17:27:28.845402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glove_path = './glove.840B.300d/glove.840B.300d.txt'\nparagram_path = './paragram_300_sl999/paragram_300_sl999.txt'\nwiki_path = './wiki-news-300d-1M/wiki-news-300d-1M.vec'","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.846556Z","iopub.status.idle":"2022-01-06T17:27:28.846946Z","shell.execute_reply.started":"2022-01-06T17:27:28.846730Z","shell.execute_reply":"2022-01-06T17:27:28.846751Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nglove_embed = load_embed(glove_path)\nprint(\"\\033[1mGlove Coverage: \\033[0m]\")\noov_glove = check_voc(vocabulary2, glove_embed)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.848449Z","iopub.status.idle":"2022-01-06T17:27:28.848823Z","shell.execute_reply.started":"2022-01-06T17:27:28.848623Z","shell.execute_reply":"2022-01-06T17:27:28.848642Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = None\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print('Use TPU')\nexcept ValueError:\n    if len(tf.config.list_physical_devices('GPU')) > 0:\n        strategy = tf.distribute.MirroredStrategy()\n        print('Use GPU')\n    else:\n        strategy = tf.distribute.get_strategy()\n        print('Use CPU')","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.850196Z","iopub.status.idle":"2022-01-06T17:27:28.850538Z","shell.execute_reply.started":"2022-01-06T17:27:28.850363Z","shell.execute_reply":"2022-01-06T17:27:28.850383Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count=0\nglove_embedding_mat=np.zeros((vocab_size,300))\nfor word,i in tqdm(word_index.items()):\n    try:\n        vec=glove_embed[word]\n        glove_embedding_mat[i]=vec\n    except KeyError:\n        count+=1\n        continue\n\nprint(\"Number of Out of Vocabulary\",count)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.852030Z","iopub.status.idle":"2022-01-06T17:27:28.852369Z","shell.execute_reply.started":"2022-01-06T17:27:28.852197Z","shell.execute_reply":"2022-01-06T17:27:28.852217Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    glove_model = get_model(glove_embedding_mat)\n    glove_model.compile(loss=bin_loss, optimizer=Adam(lr=0.001), metrics=['accuracy'])\nglove_history=glove_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.853744Z","iopub.status.idle":"2022-01-06T17:27:28.854119Z","shell.execute_reply.started":"2022-01-06T17:27:28.853943Z","shell.execute_reply":"2022-01-06T17:27:28.853965Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for loss\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 1)\nplt.plot(glove_history.history['loss'])\nplt.plot(glove_history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\n# plt.show()\n# summarize history for accuracy\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 2)\nplt.plot(glove_history.history['accuracy'])\nplt.plot(glove_history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.855527Z","iopub.status.idle":"2022-01-06T17:27:28.855915Z","shell.execute_reply.started":"2022-01-06T17:27:28.855700Z","shell.execute_reply":"2022-01-06T17:27:28.855721Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glove_y_pre=glove_model.predict(val_padded, verbose=1)\nbest_score = 0\nbest_thresh = 0\nfor thresh in np.arange(0.1,0.5,0.01):\n    if(best_score < metrics.f1_score(val.target,(glove_y_pre>thresh).astype(int))):\n        best_score = metrics.f1_score(val.target,(glove_y_pre>thresh))\n        best_thresh = round(thresh, 2)\n    print(\"threshold {0:2.2f} f1 score:{1:2.3f}\".format(thresh,metrics.f1_score(val.target,(glove_y_pre>thresh).astype(int))))\nprint(\"\\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\\033[0m\".format(best_score, best_thresh))","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.857304Z","iopub.status.idle":"2022-01-06T17:27:28.857646Z","shell.execute_reply.started":"2022-01-06T17:27:28.857463Z","shell.execute_reply":"2022-01-06T17:27:28.857479Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold=best_thresh\nglove_y_test_pre=glove_model.predict(test_padded, batch_size=BATCH_SIZE, verbose=1)\nglove_y_test_pre=(glove_y_test_pre>thresh).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.859436Z","iopub.status.idle":"2022-01-06T17:27:28.859773Z","shell.execute_reply.started":"2022-01-06T17:27:28.859614Z","shell.execute_reply":"2022-01-06T17:27:28.859629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del glove_embed, glove_embedding_mat, glove_history, best_score, best_thresh\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.860470Z","iopub.status.idle":"2022-01-06T17:27:28.860833Z","shell.execute_reply.started":"2022-01-06T17:27:28.860630Z","shell.execute_reply":"2022-01-06T17:27:28.860650Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = None\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print('Use TPU')\nexcept ValueError:\n    if len(tf.config.list_physical_devices('GPU')) > 0:\n        strategy = tf.distribute.MirroredStrategy()\n        print('Use GPU')\n    else:\n        strategy = tf.distribute.get_strategy()\n        print('Use CPU')","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.862094Z","iopub.status.idle":"2022-01-06T17:27:28.862428Z","shell.execute_reply.started":"2022-01-06T17:27:28.862254Z","shell.execute_reply":"2022-01-06T17:27:28.862276Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nparagram_embed = load_embed(paragram_path)\nprint(\"\\033[1mParagram Coverage: \\033[0m]\")\noov_paragram = check_voc(vocabulary2, paragram_embed)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.863489Z","iopub.status.idle":"2022-01-06T17:27:28.863857Z","shell.execute_reply.started":"2022-01-06T17:27:28.863652Z","shell.execute_reply":"2022-01-06T17:27:28.863674Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count=0\npara_embedding_mat=np.zeros((vocab_size,300))\nfor word,i in tqdm(word_index.items()):\n    try:\n        vec=paragram_embed[word.lower()]\n        para_embedding_mat[i]=vec\n    except KeyError:\n        count+=1\n        continue\n\nprint(\"Number of Out of Vocabulary\",count)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.865072Z","iopub.status.idle":"2022-01-06T17:27:28.865400Z","shell.execute_reply.started":"2022-01-06T17:27:28.865225Z","shell.execute_reply":"2022-01-06T17:27:28.865247Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    para_model = get_model(para_embedding_mat)\n    para_model.compile(loss=bin_loss, optimizer=Adam(lr=0.001), metrics=['accuracy'])\npara_history=para_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.867270Z","iopub.status.idle":"2022-01-06T17:27:28.867622Z","shell.execute_reply.started":"2022-01-06T17:27:28.867435Z","shell.execute_reply":"2022-01-06T17:27:28.867459Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for loss\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 1)\nplt.plot(para_history.history['loss'])\nplt.plot(para_history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\n# plt.show()\n# summarize history for accuracy\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 2)\nplt.plot(para_history.history['accuracy'])\nplt.plot(para_history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.868882Z","iopub.status.idle":"2022-01-06T17:27:28.869235Z","shell.execute_reply.started":"2022-01-06T17:27:28.869059Z","shell.execute_reply":"2022-01-06T17:27:28.869077Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"para_y_pre=para_model.predict(val_padded, verbose=1)\nbest_score = 0\nbest_thresh = 0\nfor thresh in np.arange(0.1,0.5,0.01):\n    if(best_score < metrics.f1_score(val.target,(para_y_pre>thresh).astype(int))):\n        best_score = metrics.f1_score(val.target,(para_y_pre>thresh))\n        best_thresh = round(thresh, 2)\n    print(\"threshold {0:2.2f} f1 score:{1:2.3f}\".format(thresh,metrics.f1_score(val.target,(para_y_pre>thresh).astype(int))))\nprint(\"\\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\\033[0m\".format(best_score, best_thresh))","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.870145Z","iopub.status.idle":"2022-01-06T17:27:28.870483Z","shell.execute_reply.started":"2022-01-06T17:27:28.870307Z","shell.execute_reply":"2022-01-06T17:27:28.870330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold=best_thresh\npara_y_test_pre=para_model.predict(test_padded, batch_size=BATCH_SIZE, verbose=1)\npara_y_test_pre=(para_y_test_pre>thresh).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.871352Z","iopub.status.idle":"2022-01-06T17:27:28.871669Z","shell.execute_reply.started":"2022-01-06T17:27:28.871497Z","shell.execute_reply":"2022-01-06T17:27:28.871519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del paragram_embed, para_embedding_mat, para_history, best_score, best_thresh\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.873453Z","iopub.status.idle":"2022-01-06T17:27:28.873810Z","shell.execute_reply.started":"2022-01-06T17:27:28.873615Z","shell.execute_reply":"2022-01-06T17:27:28.873636Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"strategy = None\n\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n    strategy = tf.distribute.TPUStrategy(tpu)\n    print('Use TPU')\nexcept ValueError:\n    if len(tf.config.list_physical_devices('GPU')) > 0:\n        strategy = tf.distribute.MirroredStrategy()\n        print('Use GPU')\n    else:\n        strategy = tf.distribute.get_strategy()\n        print('Use CPU')","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.875991Z","iopub.status.idle":"2022-01-06T17:27:28.876300Z","shell.execute_reply.started":"2022-01-06T17:27:28.876141Z","shell.execute_reply":"2022-01-06T17:27:28.876157Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwiki_embed = load_embed(wiki_path)\nprint(\"\\033[1mWiki Coverage: \\033[0m]\")\noov_wiki = check_voc(vocabulary2, wiki_embed)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.877327Z","iopub.status.idle":"2022-01-06T17:27:28.877635Z","shell.execute_reply.started":"2022-01-06T17:27:28.877469Z","shell.execute_reply":"2022-01-06T17:27:28.877483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"count=0\nwiki_embedding_mat=np.zeros((vocab_size,300))\nfor word,i in tqdm(word_index.items()):\n    try:\n        vec=wiki_embed[word]\n        wiki_embedding_mat[i]=vec\n    except KeyError:\n        count+=1\n        continue\n\nprint(\"Number of Out of Vocabulary\",count)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.878658Z","iopub.status.idle":"2022-01-06T17:27:28.878977Z","shell.execute_reply.started":"2022-01-06T17:27:28.878818Z","shell.execute_reply":"2022-01-06T17:27:28.878834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    wiki_model = get_model(wiki_embedding_mat)\n    wiki_model.compile(loss=bin_loss, optimizer=Adam(lr=0.001), metrics=['accuracy'])\nwiki_history=wiki_model.fit(train_padded, train.target, batch_size=BATCH_SIZE, epochs=30, validation_data=(val_padded, val.target),callbacks=my_callbacks)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.880051Z","iopub.status.idle":"2022-01-06T17:27:28.880549Z","shell.execute_reply.started":"2022-01-06T17:27:28.880330Z","shell.execute_reply":"2022-01-06T17:27:28.880360Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# summarize history for loss\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 1)\nplt.plot(wiki_history.history['loss'])\nplt.plot(wiki_history.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\n# plt.show()\n# summarize history for accuracy\nplt.figure(figsize=(15, 5)).add_subplot(1, 2, 2)\nplt.plot(wiki_history.history['accuracy'])\nplt.plot(wiki_history.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.881510Z","iopub.status.idle":"2022-01-06T17:27:28.882002Z","shell.execute_reply.started":"2022-01-06T17:27:28.881801Z","shell.execute_reply":"2022-01-06T17:27:28.881826Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"wiki_y_pre=wiki_model.predict(val_padded, verbose=1)\nbest_score = 0\nbest_thresh = 0\nfor thresh in np.arange(0.1,0.5,0.01):\n    if(best_score < metrics.f1_score(val.target,(wiki_y_pre>thresh).astype(int))):\n        best_score = metrics.f1_score(val.target,(wiki_y_pre>thresh))\n        best_thresh = round(thresh, 2)\n    print(\"threshold {0:2.2f} f1 score:{1:2.3f}\".format(thresh,metrics.f1_score(val.target,(wiki_y_pre>thresh).astype(int))))\nprint(\"\\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\\033[0m\".format(best_score, best_thresh))","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.882776Z","iopub.status.idle":"2022-01-06T17:27:28.883427Z","shell.execute_reply.started":"2022-01-06T17:27:28.883208Z","shell.execute_reply":"2022-01-06T17:27:28.883232Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold=best_thresh\nwiki_y_test_pre=wiki_model.predict(test_padded, batch_size=BATCH_SIZE, verbose=1)\nwiki_y_test_pre=(wiki_y_test_pre>thresh).astype(int)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.884163Z","iopub.status.idle":"2022-01-06T17:27:28.884466Z","shell.execute_reply.started":"2022-01-06T17:27:28.884305Z","shell.execute_reply":"2022-01-06T17:27:28.884322Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del wiki_embed, wiki_embedding_mat, wiki_history, best_score, best_thresh\ngc.collect()","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.885276Z","iopub.status.idle":"2022-01-06T17:27:28.885603Z","shell.execute_reply.started":"2022-01-06T17:27:28.885424Z","shell.execute_reply":"2022-01-06T17:27:28.885445Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pre=0.25*(google_y_pre + glove_y_pre + para_y_pre + wiki_y_pre)\n# y_pre=0.20*google_y_pre + 0.35*glove_y_pre + 0.15*para_y_pre + 0.30*wiki_y_pre\nbest_score = 0\nbest_thresh = 0\nfor thresh in np.arange(0.1,0.5,0.01):\n    if(best_score < metrics.f1_score(val.target,(y_pre>thresh).astype(int))):\n        best_score = metrics.f1_score(val.target,(y_pre>thresh))\n        best_thresh = round(thresh, 2)\n    print(\"threshold {0:2.2f} f1 score:{1:2.3f}\".format(thresh,metrics.f1_score(val.target,(y_pre>thresh).astype(int))))\nprint(\"\\033[1mBest result {0:2.3f} in thresh_hold {1:2.2f}\\033[0m\".format(best_score, best_thresh))","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.887741Z","iopub.status.idle":"2022-01-06T17:27:28.888280Z","shell.execute_reply.started":"2022-01-06T17:27:28.888059Z","shell.execute_reply":"2022-01-06T17:27:28.888090Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# y_test_pre = 0.25 * (google_y_test_pre + glove_y_test_pre + para_y_test_pre + wiki_y_test_pre)\ny_test_pre = 0.2*google_y_test_pre + 0.3*glove_y_test_pre + 0.2*para_y_test_pre + 0.3*wiki_y_test_pre\ny_test_pre=(y_test_pre>thresh).astype(int)\n### Tạo File submission\nsubmit=pd.DataFrame()\nsubmit[\"qid\"]=df_test.qid\nsubmit[\"prediction\"]=y_test_pre\nsubmit.to_csv(\"submission.csv\",index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-06T17:27:28.889271Z","iopub.status.idle":"2022-01-06T17:27:28.889776Z","shell.execute_reply.started":"2022-01-06T17:27:28.889551Z","shell.execute_reply":"2022-01-06T17:27:28.889578Z"},"trusted":true},"execution_count":null,"outputs":[]}]}