{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**BÀI TẬP LỚN  MÔN HỌC MÁY**\n\n19021374 - Phạm Thị Minh Trang\n","metadata":{}},{"cell_type":"markdown","source":"# Mô tả bài toán\nPhân loại các câu hỏi thật (sincere) và câu hỏi chỉ mang tính câu fame (insincere).\n\n**Input**: Các câu hỏi ở dạng text\n\n**Output**: 0 nếu câu hỏi là sincere và 1 nếu là insincere.","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2022-01-11T06:52:46.307724Z","iopub.execute_input":"2022-01-11T06:52:46.30811Z","iopub.status.idle":"2022-01-11T06:52:46.339878Z","shell.execute_reply.started":"2022-01-11T06:52:46.308002Z","shell.execute_reply":"2022-01-11T06:52:46.339181Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Phân tích dữ liệu\nDữ liệu được cung cấp sẵn gồm có hai file csv để lưu dữ liệu train và test, một file zip chứa 4 loại pretrained word embeddings.","metadata":{}},{"cell_type":"code","source":"train_path = '../input/quora-insincere-questions-classification/train.csv'\ntest_path = '../input/quora-insincere-questions-classification/test.csv'\nemb_path = '../input/quora-insincere-questions-classification/embeddings.zip'\n\ntrain_data = pd.read_csv(train_path)\ntest_data = pd.read_csv(test_path)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:52:46.341247Z","iopub.execute_input":"2022-01-11T06:52:46.341752Z","iopub.status.idle":"2022-01-11T06:52:52.675958Z","shell.execute_reply.started":"2022-01-11T06:52:46.341715Z","shell.execute_reply":"2022-01-11T06:52:52.675Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dữ liệu huấn luyện gồm có ba trường: \n- qid: đánh index định danh cho câu hỏi. Không ảnh hưởng gì đến việc huấn luyện.\n- question_text: nội dung câu hỏi cần phải phân loại.\n- target: phân loại của câu hỏi trên. Giá trị 0 là sincere, 1 là insincere.","metadata":{}},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:52:52.677144Z","iopub.execute_input":"2022-01-11T06:52:52.677416Z","iopub.status.idle":"2022-01-11T06:52:52.697197Z","shell.execute_reply.started":"2022-01-11T06:52:52.677364Z","shell.execute_reply":"2022-01-11T06:52:52.696252Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Tập dữ liệu huấn luyện có 1306122 mẫu, được phân vào 2 lớp (0 và 1). Trong đó, có 1225312 thuộc lớp 0, 80810 thuộc lớp 1.\n\nTrung bình một mẫu có độ dài 11 từ","metadata":{}},{"cell_type":"code","source":"# số lượng các mẫu trong tập huấn luyện\nnum_sample = len(train_data)\nprint('Number of training samples: ', num_sample)\nprint('Number of classes', max(train_data['target'])+1)\n\nnum_word = [len(s.split()) for s in train_data['question_text']]\n# trung bình số từ trong một mẫu\nnum_wps = np.median(num_word)\nprint('Number of words per sample: ', num_wps)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:52:52.69979Z","iopub.execute_input":"2022-01-11T06:52:52.700124Z","iopub.status.idle":"2022-01-11T06:52:54.52636Z","shell.execute_reply.started":"2022-01-11T06:52:52.70008Z","shell.execute_reply":"2022-01-11T06:52:54.525444Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data['target'].value_counts()","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:52:54.527845Z","iopub.execute_input":"2022-01-11T06:52:54.528272Z","iopub.status.idle":"2022-01-11T06:52:54.54732Z","shell.execute_reply.started":"2022-01-11T06:52:54.528224Z","shell.execute_reply":"2022-01-11T06:52:54.546441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Một cái biểu đồ đẹp đẽ thể hiện sự mất cân bằng giữa hai phân lớp trong tập dữ liệu train\nimport matplotlib.pyplot as plt\n\nplt.figure(figsize=(10,7))\nplt.subplot(1, 2, 1)\nplt.title('Question count')\nbar = plt.bar(['Sincere', 'Insincere'], [train_data.loc[train_data['target'] == 0].count()[0], train_data.loc[train_data['target'] == 1].count()[0]])","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:52:54.548798Z","iopub.execute_input":"2022-01-11T06:52:54.54906Z","iopub.status.idle":"2022-01-11T06:52:55.237262Z","shell.execute_reply.started":"2022-01-11T06:52:54.549028Z","shell.execute_reply":"2022-01-11T06:52:55.236434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# # cắt bớt data để 2 nhóm cân bằng\n# from random import sample\n# train_data_balance = train_data[train_data['target'] == 1]\n# train_data_balance = train_data_balance.append(train_data[train_data['target'] == 0].sample(len(train_data[train_data['target']==1])))\\n\n# # shuffle all rows and reset index columns\n# train_data_balance = train_data_balance.sample(frac=1).reset_index(drop=True)\n# train_data_balance","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:52:55.238678Z","iopub.execute_input":"2022-01-11T06:52:55.239003Z","iopub.status.idle":"2022-01-11T06:52:55.243301Z","shell.execute_reply.started":"2022-01-11T06:52:55.238968Z","shell.execute_reply":"2022-01-11T06:52:55.242238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Làm sạch dữ liệu\n- Xóa ngắt câu và kí tự lạ\n- Xóa số\n- Sửa sai chính tả\n- Viết thường","metadata":{}},{"cell_type":"code","source":"punctuation_list = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', \n        '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', \n        '█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', \n        '—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞', \n        '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━', \n        '⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○', \n        '◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐', \n        '├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\\x97', '↺', \n        '∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊', \n        '┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´', \n        '↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\\x93', '≧', '］', \n        '\\x94', '∀', '♛', '\\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％', \n        '╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭', \n        '┊', '╯', '☾', '➔', '∴', '\\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐', \n        '▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\\x91', '➚', '◡', '╰', '\\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒', \n        '☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜', \n        '‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢', \n        '≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥', \n        '⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ', \n        '✎', ';', '␤', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦', \n        '◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ', \n        '⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋', \n        '､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞', \n        '◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ', \n        '✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱', \n        '☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹', \n        '⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚', \n        '☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛', \n        '⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙', \n        'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘', \n        '˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛', \n        '☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑', \n        '✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ', \n        '⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘', \n        '⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊', \n        '⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷', \n        '◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝', \n        '⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢', \n        '≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']\n\n# Chứa cả các từ viết sai chính tả, các biến thể us uk, và các từ viết tắt\nmisspell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', \n                 'counselling': 'counseling','theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', \n                 'organisation': 'organization', 'wwii': 'world war 2','citicise': 'criticize', 'youtu ': 'youtube ', \n                 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist',\n                 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', \n                 'whydo': 'why do','doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', \n                 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'bitcoin', \n                 'narcissit': 'narcissist', 'bigdata': 'big data','2k17': '2017', '2k18': '2018', 'qouta': 'quota', \n                 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what','watsapp': 'whatsapp', \n                 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization',                 'electroneum':'bitcoin','nanodegree':'degree','hotstar':'star','dream11':'dream','ftre':'fire','tensorflow':'framework',                'unocoin':'bitcoin','lnmiit':'limit','unacademy':'academy','altcoin':'bitcoin','altcoins':'bitcoin','litecoin':'bitcoin',                'coinbase':'bitcoin','cryptocurency':'cryptocurrency','simpliv':'simple','quoras':'quora','schizoids':'psychopath',                'remainers':'remainder','twinflame':'soulmate','quorans':'quora','brexit':'demonetized','iiest':'institute','dceu':'comics',                'pessat':'exam','uceed':'college','bhakts':'devotee','boruto':'anime','cryptocoin':'bitcoin','blockchains':'blockchain',                'fiancee':'fiance','redmi':'smartphone','oneplus':'smartphone','qoura':'quora','deepmind':'framework','ryzen':'cpu',                'whattsapp':'whatsapp','undertale':'adventure','zenfone':'smartphone','cryptocurencies':'cryptocurrencies','koinex':'bitcoin',                'zebpay':'bitcoin','binance':'bitcoin','whtsapp':'whatsapp','reactjs':'framework','bittrex':'bitcoin','bitconnect':'bitcoin',                'bitfinex':'bitcoin','yourquote':'your quote','whyis':'why is','jiophone':'smartphone','dogecoin':'bitcoin','onecoin':'bitcoin',                'poloniex':'bitcoin','7700k':'cpu','angular2':'framework','segwit2x':'bitcoin','hashflare':'bitcoin','940mx':'gpu',                'openai':'framework','hashflare':'bitcoin','1050ti':'gpu','nearbuy':'near buy','freebitco':'bitcoin','antminer':'bitcoin',                'filecoin':'bitcoin','whatapp':'whatsapp', 'empowr':'empower','1080ti':'gpu','crytocurrency':'cryptocurrency','8700k':'cpu',                'whatsaap':'whatsapp','g4560':'cpu','payymoney':'pay money','fuckboys':'fuck boys','intenship':'internship','zcash':'bitcoin',                'demonatisation':'demonetization','narcicist':'narcissist','mastuburation':'masturbation','trignometric':'trigonometric',                'cryptocurreny':'cryptocurrency','howdid':'how did','crytocurrencies':'cryptocurrencies','phycopath':'psychopath',                'bytecoin':'bitcoin','possesiveness':'possessiveness','scollege':'college','humanties':'humanities','altacoin':'bitcoin',                'demonitised':'demonetized','brasília':'brazilia','accolite':'accolyte','econimics':'economics','varrier':'warrier','quroa':'quora',                'statergy':'strategy','langague':'language', 'splatoon':'game','7600k':'cpu','gate2018':'gate 2018','in2018':'in 2018',                'narcassist':'narcissist','jiocoin':'bitcoin','hnlu':'hulu','7300hq':'cpu','weatern':'western','interledger':'blockchain',                'deplation':'deflation', 'cryptocurrencies':'cryptocurrency', 'bitcoin':'blockchain cryptocurrency',\n                \"ain't\": \"is not\", \"aren't\": \"are not\", \"can't\": \"cannot\", \"'cause\": \"because\",\n                \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\", \"doesn't\": \"does not\",\n                \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n                \"he'd\": \"he would\", \"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\",\n                \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\", \"i'd\": \"i would\",\n                \"i'd've\": \"i would have\", \"i'll\": \"i will\", \"i'll've\": \"I will have\", \"i'm\": \"i am\",\n                \"i've\": \"I have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n                \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\", \"it's\": \"it is\",\n                \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\n                \"mightn't\": \"might not\", \"mightn't've\": \"might not have\", \"must've\": \"must have\",\n                \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\",\n                \"needn't've\": \"need not have\", \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\",\n                \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\",\n                \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\",\n                \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n                \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\",\n                \"so've\": \"so have\", \"so's\": \"so as\", \"this's\": \"this is\", \"that'd\": \"that would\",\n                \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n                \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\n                \"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\",\n                \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\",\n                \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\",\n                \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\",\n                \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\",\n                \"what're\": \"what are\", \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\",\n                \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\",\n                \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n                \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\",\n                \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\",\n                \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n                \"y'all'd've\": \"you all would have\", \"y'all're\": \"you all are\", \"y'all've\": \"you all have\",\n                \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\",\n                \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\"}               \n","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:52:55.245153Z","iopub.execute_input":"2022-01-11T06:52:55.245599Z","iopub.status.idle":"2022-01-11T06:52:55.425922Z","shell.execute_reply.started":"2022-01-11T06:52:55.245543Z","shell.execute_reply":"2022-01-11T06:52:55.425005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import re\n\ndef clear_punct(text):\n    for p in punctuation_list:\n        if p in text:\n            text = text.replace(p, \" \")\n    return text\nprint(clear_punct(\"Ha♌hEha.!fsd\"))\n\ndef replace_word(text):\n    for i in misspell_dict:\n        if i in text:\n            text = text.replace(i, misspell_dict[i])\n    return text\nprint(replace_word(\"you're my favourite narcicist\"))\n                   \ndef clear_num(text):\n    text = re.sub('[0-9]{5,}', '#####', text)\n    text = re.sub('[0-9]{4}', '####', text)\n    text = re.sub('[0-9]{3}', '###', text)\n    text = re.sub('[0-9]{2}', '##', text)\n    return text\nprint(clear_num(\"10 little indians\"))","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:52:55.427218Z","iopub.execute_input":"2022-01-11T06:52:55.427501Z","iopub.status.idle":"2022-01-11T06:52:55.4452Z","shell.execute_reply.started":"2022-01-11T06:52:55.427469Z","shell.execute_reply":"2022-01-11T06:52:55.444368Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# thực hiện các làm làm sạch sử liệu train và test\ntrain_data['question_text'] = train_data['question_text'].apply(lambda x: x.lower())\ntrain_data['question_text'] = train_data['question_text'].apply(lambda x: clear_num(x))\ntrain_data['question_text'] = train_data['question_text'].apply(lambda x: replace_word(x))\ntrain_data['question_text'] = train_data['question_text'].apply(lambda x: clear_punct(x))\n\ntest_data['question_text'] = test_data['question_text'].apply(lambda x: x.lower())\ntest_data['question_text'] = test_data['question_text'].apply(lambda x: clear_num(x))\ntest_data['question_text'] = test_data['question_text'].apply(lambda x: replace_word(x))\ntest_data['question_text'] = test_data['question_text'].apply(lambda x: clear_punct(x))","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:52:55.449017Z","iopub.execute_input":"2022-01-11T06:52:55.449598Z","iopub.status.idle":"2022-01-11T06:55:22.184854Z","shell.execute_reply.started":"2022-01-11T06:52:55.449557Z","shell.execute_reply":"2022-01-11T06:55:22.183791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# tách dữ liệu train thành 2 phần nhỏ hơn là tập train và tập validation kiểm thử\nfrom sklearn.model_selection import train_test_split\n\ntrain_x, val_x, train_y, val_y = train_test_split(train_data['question_text'].values, train_data['target'].values, random_state = 1)\ntest_x = test_data['question_text'].values","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:55:22.187079Z","iopub.execute_input":"2022-01-11T06:55:22.187326Z","iopub.status.idle":"2022-01-11T06:55:23.499073Z","shell.execute_reply.started":"2022-01-11T06:55:22.187298Z","shell.execute_reply":"2022-01-11T06:55:23.498015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Xử lý dữ liệu\n- Tokenize văn bản thành các từ\n- Tạo bộ vocab sử dụng top 20000 token \n- Vector hóa các token\n- Pad (hoặc truncate) để cho các câu đều có độ dài bằng nhau","metadata":{}},{"cell_type":"code","source":"from tensorflow.python.keras.preprocessing import sequence\nfrom tensorflow.python.keras.preprocessing import text\n\n# mỗi mẫu có 100 từ\nMAX_LENGTH = 100\n\n# Tạo bộ vocab với dữ liệu train\ntokenizer = text.Tokenizer(num_words=20000)\ntokenizer.fit_on_texts(list(train_x))\n\n# Chuyển thành các sequence vector\ntrain_x = tokenizer.texts_to_sequences(train_x)\nval_x = tokenizer.texts_to_sequences(val_x)\ntest_x = tokenizer.texts_to_sequences(test_x)\n\n# Pad độ dài các mẫu\ntrain_x = sequence.pad_sequences(train_x, maxlen=MAX_LENGTH)\nval_x = sequence.pad_sequences(val_x, maxlen=MAX_LENGTH)\ntest_x = sequence.pad_sequences(test_x, maxlen=MAX_LENGTH)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:55:23.500413Z","iopub.execute_input":"2022-01-11T06:55:23.500648Z","iopub.status.idle":"2022-01-11T06:56:36.190506Z","shell.execute_reply.started":"2022-01-11T06:55:23.500619Z","shell.execute_reply":"2022-01-11T06:56:36.189455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Embeddings**\n\nSử dụng 3 bộ pretrained embeddings mà đề bài đã cho sẵn: glove, wiki-news, paragram","metadata":{}},{"cell_type":"code","source":"from zipfile import ZipFile\n\nzip_file = ZipFile(emb_path, 'r')\nzip_file.extractall()\nzip_file.namelist()","metadata":{"execution":{"iopub.status.busy":"2022-01-11T06:56:36.191721Z","iopub.execute_input":"2022-01-11T06:56:36.191964Z","iopub.status.idle":"2022-01-11T07:01:22.375106Z","shell.execute_reply.started":"2022-01-11T06:56:36.191935Z","shell.execute_reply":"2022-01-11T07:01:22.373697Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glove_path = './glove.840B.300d/glove.840B.300d.txt'\nwiki_path = './wiki-news-300d-1M/wiki-news-300d-1M.vec'\nparagram_path =  './paragram_300_sl999/paragram_300_sl999.txt'","metadata":{"execution":{"iopub.status.busy":"2022-01-11T07:01:22.377149Z","iopub.execute_input":"2022-01-11T07:01:22.37745Z","iopub.status.idle":"2022-01-11T07:01:22.383882Z","shell.execute_reply.started":"2022-01-11T07:01:22.377418Z","shell.execute_reply":"2022-01-11T07:01:22.383256Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_features = 20000\nemb_dim = 300","metadata":{"execution":{"iopub.status.busy":"2022-01-11T07:01:22.385011Z","iopub.execute_input":"2022-01-11T07:01:22.385472Z","iopub.status.idle":"2022-01-11T07:01:23.292673Z","shell.execute_reply.started":"2022-01-11T07:01:22.385426Z","shell.execute_reply":"2022-01-11T07:01:23.29127Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# https://www.kaggle.com/gmhost/gru-capsule\ndef load_emb(word_index, emb_path, is_parag=False):\n    def get_coefs(word,*arr): \n        return word, np.asarray(arr, dtype='float32')\n    # paragram uses utf8, different from others -> this function looks stupid\n    if is_parag:\n        emb_index = dict(get_coefs(*o.split(' ')) for o in open(emb_path, encoding='utf8', errors='ignore') if len(o)>100 and o.split(' ')[0] in word_index)\n    else:\n        emb_index = dict(get_coefs(*o.split(' ')) for o in open(emb_path) if len(o)>100 and o.split(' ')[0] in word_index )\n    all_embs = np.stack(emb_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    emb_matrix = np.random.normal(emb_mean, emb_std, (num_features, embed_size))\n    for word, i in word_index.items():\n        if i >= num_features: continue\n        emb_vector = emb_index.get(word)\n        if emb_vector is not None: emb_matrix[i] = emb_vector\n    return emb_matrix","metadata":{"execution":{"iopub.status.busy":"2022-01-11T07:01:23.294184Z","iopub.execute_input":"2022-01-11T07:01:23.294806Z","iopub.status.idle":"2022-01-11T07:01:23.311461Z","shell.execute_reply.started":"2022-01-11T07:01:23.294767Z","shell.execute_reply":"2022-01-11T07:01:23.310377Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"glove_emb = load_emb(tokenizer.word_index, glove_path)\nwiki_emb = load_emb(tokenizer.word_index, wiki_path)\nparag_emb = load_emb(tokenizer.word_index, paragram_path, True)\n# Sử dụng bộ embedding là trung bình của cả 3 bộ trên\nemb_matrix = np.mean([glove_emb, wiki_emb, parag_emb], axis = 0)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T07:01:23.312899Z","iopub.execute_input":"2022-01-11T07:01:23.313879Z","iopub.status.idle":"2022-01-11T07:04:23.935697Z","shell.execute_reply.started":"2022-01-11T07:01:23.313826Z","shell.execute_reply":"2022-01-11T07:04:23.934819Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Mô hình\nXây dựng theo mô hình RNN (https://www.tensorflow.org/text/tutorials/text_classification_rnn#create_the_model)\n\nVề RNN: https://tricky-tax-444.notion.site/06-M-ng-n-ron-cho-chu-i-d-li-u-6e64974cae7a4bf098c98d196dc6534e#b599baf0f61744678f992007b0f01034\n\n![](https://www.tensorflow.org/text/tutorials/images/bidirectional.png)\n\nXây dưng một mô hình đơn giản gồm các lớp:\n- Embedding: sử dụng bộ embedding đã tính phía trên, freeze lớp này lại, vì em thấy nếu không thì sẽ khá tốn thời gian và tốn tài nguyên mà cũng không đem lại hiệu quả cao hơn mấy.\n- Bidirectional: chỉ sử dụng như một lớp ở trong mô hình\n- MaxPool\n- Dense\n- Dropout","metadata":{}},{"cell_type":"code","source":"from tensorflow.python.keras import models\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.python.keras.layers import Dense\nfrom tensorflow.python.keras.layers import Input\nfrom tensorflow.python.keras.layers import Dropout\nfrom tensorflow.python.keras.layers import Embedding\nfrom tensorflow.python.keras.layers import Bidirectional\nfrom tensorflow.python.keras.layers import LSTM\nfrom tensorflow.python.keras.layers import Conv1D\nfrom tensorflow.python.keras.layers import GlobalMaxPool1D\nfrom tensorflow.python.keras.layers import GRU\n\n\ninputs = Input(shape=(MAX_LENGTH))\nx = Embedding(input_dim=num_features, output_dim=emb_dim,weights=[emb_matrix], trainable=False)(inputs)\nx = Bidirectional(GRU(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation='relu')(x)\nx = Dropout(0.1)(x)\noutputs = Dense(1, activation='sigmoid')(x)\n\nmodel = models.Model(inputs=inputs, outputs=outputs)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\ncallbacks = [EarlyStopping(monitor='val_loss', patience=2)]\n\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2022-01-11T07:04:23.937058Z","iopub.execute_input":"2022-01-11T07:04:23.937304Z","iopub.status.idle":"2022-01-11T07:04:25.777887Z","shell.execute_reply.started":"2022-01-11T07:04:23.937271Z","shell.execute_reply":"2022-01-11T07:04:25.776883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"history = model.fit(train_x,train_y, epochs = 5, validation_data = (val_x,val_y), callbacks = callbacks, verbose = 1,batch_size = 256)\n# Print results.\nhistory = history.history\n# print('Validation accuracy: {acc}, loss: {loss}'.format(acc=history['val_acc'][-1], loss=history['val_loss'][-1]))","metadata":{"execution":{"iopub.status.busy":"2022-01-11T07:04:25.779159Z","iopub.execute_input":"2022-01-11T07:04:25.779552Z","iopub.status.idle":"2022-01-11T11:56:33.856947Z","shell.execute_reply.started":"2022-01-11T07:04:25.779516Z","shell.execute_reply":"2022-01-11T11:56:33.855123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pred = model.predict(test_x, batch_size = 256)\ny_test = (pred[:,0] > 0.5).astype(np.int)\nsubmission = pd.DataFrame({'qid': test_data['qid'], 'prediction': y_test})\nsubmission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-11T12:08:05.322251Z","iopub.execute_input":"2022-01-11T12:08:05.322586Z","iopub.status.idle":"2022-01-11T12:13:44.874788Z","shell.execute_reply.started":"2022-01-11T12:08:05.322549Z","shell.execute_reply":"2022-01-11T12:13:44.873629Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}