{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"**GIỚI THIỆU VỀ TẬP DỮ LIỆU**\n\nQuora là một nền tảng cho phép mọi người học hỏi lẫn nhau. Trên Quora, mọi người có thể đặt câu hỏi và kết nối với những người khác, những người đóng góp thông tin chi tiết độc đáo và câu trả lời chất lượng. Một thách thức quan trọng là loại bỏ những câu hỏi thiếu chân thành - những câu hỏi được đặt ra dựa trên những tiền đề sai lầm hoặc có ý định đưa ra một tuyên bố hơn là tìm kiếm những câu trả lời hữu ích.\n\nTrong cuộc thi này, Kagglers sẽ phát triển các mô hình xác định và gắn cờ cho các câu hỏi không chân thành.\n\n**Mô tả tệp**\n\n* train.csv - tập huấn luyện\n* test.csv - bộ thử nghiệm\n\n\n**Các trường dữ liệu**\n\n* qid - mã định danh câu hỏi duy nhất\n* question_text - câu hỏi Quora\n* target - câu hỏi có nhãn \"insincere\" có giá trị bằng 1, ngược lại bằng 0","metadata":{}},{"cell_type":"code","source":"import re\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport seaborn as sns\nfrom tqdm import tqdm\ntqdm.pandas()\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.metrics import accuracy_score, f1_score\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, GRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nimport tensorflow_addons as tfa\n\n\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.initializers import *\nfrom keras.optimizers import *\nimport keras.backend as K\nfrom keras.callbacks import *\nimport tensorflow as tf\nimport os\nimport time\nimport gc\nimport re\nimport glob","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-06-10T15:32:58.375656Z","iopub.execute_input":"2021-06-10T15:32:58.376025Z","iopub.status.idle":"2021-06-10T15:32:58.512997Z","shell.execute_reply.started":"2021-06-10T15:32:58.375993Z","shell.execute_reply":"2021-06-10T15:32:58.51219Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# DATA OVERVIEW","metadata":{}},{"cell_type":"markdown","source":"**ĐỌC DỮ LIỆU**","metadata":{}},{"cell_type":"code","source":"train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:01.213736Z","iopub.execute_input":"2021-06-10T15:10:01.214053Z","iopub.status.idle":"2021-06-10T15:10:05.511866Z","shell.execute_reply.started":"2021-06-10T15:10:01.21402Z","shell.execute_reply":"2021-06-10T15:10:05.510978Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:05.51365Z","iopub.execute_input":"2021-06-10T15:10:05.514021Z","iopub.status.idle":"2021-06-10T15:10:05.744111Z","shell.execute_reply.started":"2021-06-10T15:10:05.513985Z","shell.execute_reply":"2021-06-10T15:10:05.743201Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Nhận xét:*** Dữ liệu huấn luyện không có giá trị null","metadata":{}},{"cell_type":"code","source":"test.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:05.74582Z","iopub.execute_input":"2021-06-10T15:10:05.746178Z","iopub.status.idle":"2021-06-10T15:10:05.813764Z","shell.execute_reply.started":"2021-06-10T15:10:05.746142Z","shell.execute_reply":"2021-06-10T15:10:05.812833Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Nhận xét:*** Dữ liệu kiểm thử không có giá trị null","metadata":{}},{"cell_type":"markdown","source":"**TỈ LỆ PHÂN BỐ NHÃN TRONG TẬP DỮ LIỆU HUẤN LUYỆN**","metadata":{}},{"cell_type":"code","source":"ax, fig = plt.subplots(figsize=(10, 7))\nquestion_class = train[\"target\"].value_counts()\nquestion_class.plot(kind= 'bar', color= [\"blue\", \"orange\"])\nplt.title('Bar chart')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:05.815076Z","iopub.execute_input":"2021-06-10T15:10:05.815422Z","iopub.status.idle":"2021-06-10T15:10:05.97309Z","shell.execute_reply.started":"2021-06-10T15:10:05.815386Z","shell.execute_reply":"2021-06-10T15:10:05.972039Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Tỉ lệ phần trăm số câu hỏi Insincere là:\", (len(train.loc[train.target==1])) / (len(train.loc[train.target == 0])) * 100)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:05.974546Z","iopub.execute_input":"2021-06-10T15:10:05.974915Z","iopub.status.idle":"2021-06-10T15:10:06.242194Z","shell.execute_reply.started":"2021-06-10T15:10:05.974879Z","shell.execute_reply":"2021-06-10T15:10:06.241186Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Nhận xét***: Số câu hỏi \"insincere\" chỉ chiếm khoảng 6-7% trong tổng số câu hỏi. Dữ liệu bị mất cân bằng khá lớn, do đó độ đo F1 có vẻ thích hợp cho những trường hợp như này","metadata":{}},{"cell_type":"markdown","source":"**PHÂN TÍCH TỪNG CÂU HỎI**","metadata":{}},{"cell_type":"markdown","source":"**Số lượng từ trong câu**","metadata":{}},{"cell_type":"code","source":"words = train['question_text'].apply(lambda x: len(x) - len(''.join(x.split())) + 1)\ntrain['words'] = words\nwords = train.loc[train['words']<200]['words']\nsns.distplot(words, color='g')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:06.243551Z","iopub.execute_input":"2021-06-10T15:10:06.24391Z","iopub.status.idle":"2021-06-10T15:10:13.418258Z","shell.execute_reply.started":"2021-06-10T15:10:06.243872Z","shell.execute_reply":"2021-06-10T15:10:13.417434Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Số lượng từ trung bình của các câu hỏi trong dữ liệu huấn luyện là {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Số lượng từ trung bình của các câu hỏi trong dữ liệu kiểm thử là {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x.split())))))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:13.420891Z","iopub.execute_input":"2021-06-10T15:10:13.421219Z","iopub.status.idle":"2021-06-10T15:10:15.469421Z","shell.execute_reply.started":"2021-06-10T15:10:13.421186Z","shell.execute_reply":"2021-06-10T15:10:15.468519Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Số lượng từ lớn nhất của các câu hỏi trong dữ liệu huấn luyện là {0:.0f}.'.format(np.max(train['question_text'].apply(lambda x: len(x.split())))))\nprint('Số lượng từ lớn nhất của các câu hỏi trong dữ liệu kiểm thử là {0:.0f}.'.format(np.max(test['question_text'].apply(lambda x: len(x.split())))))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:15.471429Z","iopub.execute_input":"2021-06-10T15:10:15.471803Z","iopub.status.idle":"2021-06-10T15:10:17.497081Z","shell.execute_reply.started":"2021-06-10T15:10:15.471765Z","shell.execute_reply":"2021-06-10T15:10:17.496166Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Số lượng ký tự trung bình của các câu hỏi trong dữ liệu huấn luyện là {0:.0f}.'.format(np.mean(train['question_text'].apply(lambda x: len(x)))))\nprint('Số lượng ký tự trung bình của các câu hỏi trong dữ liệu kiểm thử là {0:.0f}.'.format(np.mean(test['question_text'].apply(lambda x: len(x)))))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:17.498519Z","iopub.execute_input":"2021-06-10T15:10:17.498885Z","iopub.status.idle":"2021-06-10T15:10:18.321501Z","shell.execute_reply.started":"2021-06-10T15:10:17.498845Z","shell.execute_reply":"2021-06-10T15:10:18.320574Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"***Nhận xét:*** Có thể thấy độ dài trung bình của các câu hỏi trong tập dữ liệu huấn luyện và kiểm thử tương tự nhau, tuy nhiên có những câu hỏi khá dài trong tập dữ liệu huấn luyện","metadata":{}},{"cell_type":"markdown","source":"# DATA PREPROCESSING","metadata":{}},{"cell_type":"markdown","source":"**CÁC BƯỚC TIỀN XỬ LÝ DỮ LIỆU:**\n* Lowering\n* Xoá các số (do các số không có ý nghĩa trong việc phân lớp)\n* Xoá các kí tự đặc biệt\n* Mispelling (thay thế các từ viết tắt)","metadata":{}},{"cell_type":"markdown","source":"Như đã phân tích về dữ liệu ở trên, ta sẽ quy chuẩn số lượng từ trong một câu hỏi. Do ta sử dụng tập từ điển đã được huấn luyện sẵn là tập embeddings nên mỗi vector có độ dài là 300, và mỗi câu hỏi sẽ có tối đa 80 từ","metadata":{}},{"cell_type":"code","source":"embed_size = 300 #độ dài của vector\nmax_features = 120000 #số lượng từ xuất hiện nhiều nhất để huấn luyện\nmaxlen = 80 #số lượng từ trong câu ","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:23:38.038812Z","iopub.execute_input":"2021-06-10T15:23:38.039145Z","iopub.status.idle":"2021-06-10T15:23:38.04361Z","shell.execute_reply.started":"2021-06-10T15:23:38.039115Z","shell.execute_reply":"2021-06-10T15:23:38.042401Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Lowering**","metadata":{}},{"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].apply(lambda x: x.lower())\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: x.lower())","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:18.340553Z","iopub.execute_input":"2021-06-10T15:10:18.340909Z","iopub.status.idle":"2021-06-10T15:10:19.056845Z","shell.execute_reply.started":"2021-06-10T15:10:18.340885Z","shell.execute_reply":"2021-06-10T15:10:19.055778Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Xoá các số**","metadata":{}},{"cell_type":"code","source":"def clean_numbers(x):\n    if bool(re.search(r'\\d', x)):\n        x = re.sub('[0-9]{5,}', '#####', x)\n        x = re.sub('[0-9]{4}', '####', x)\n        x = re.sub('[0-9]{3}', '###', x)\n        x = re.sub('[0-9]{2}', '##', x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:19.060068Z","iopub.execute_input":"2021-06-10T15:10:19.060333Z","iopub.status.idle":"2021-06-10T15:10:19.065277Z","shell.execute_reply.started":"2021-06-10T15:10:19.060305Z","shell.execute_reply":"2021-06-10T15:10:19.064353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_numbers(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:19.066544Z","iopub.execute_input":"2021-06-10T15:10:19.06705Z","iopub.status.idle":"2021-06-10T15:10:27.669161Z","shell.execute_reply.started":"2021-06-10T15:10:19.067014Z","shell.execute_reply":"2021-06-10T15:10:27.668259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Xoá ký tự đặc biệt**","metadata":{}},{"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        if punct in x:\n            x = x.replace(punct, f' {punct} ')\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:27.670552Z","iopub.execute_input":"2021-06-10T15:10:27.670964Z","iopub.status.idle":"2021-06-10T15:10:27.680512Z","shell.execute_reply.started":"2021-06-10T15:10:27.670925Z","shell.execute_reply":"2021-06-10T15:10:27.679738Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: clean_text(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_text(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:27.681844Z","iopub.execute_input":"2021-06-10T15:10:27.682399Z","iopub.status.idle":"2021-06-10T15:10:40.577738Z","shell.execute_reply.started":"2021-06-10T15:10:27.682362Z","shell.execute_reply":"2021-06-10T15:10:40.576858Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Mispelling**","metadata":{}},{"cell_type":"code","source":"mispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:40.579148Z","iopub.execute_input":"2021-06-10T15:10:40.57947Z","iopub.status.idle":"2021-06-10T15:10:40.602387Z","shell.execute_reply.started":"2021-06-10T15:10:40.579438Z","shell.execute_reply":"2021-06-10T15:10:40.601583Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: replace_typical_misspell(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:10:40.605127Z","iopub.execute_input":"2021-06-10T15:10:40.605409Z","iopub.status.idle":"2021-06-10T15:11:33.968046Z","shell.execute_reply.started":"2021-06-10T15:10:40.605384Z","shell.execute_reply":"2021-06-10T15:11:33.967151Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fill up the missing values\n# train_X = train[\"question_text\"].fillna(\"_##_\").values\n# test_X = test[\"question_text\"].fillna(\"_##_\").values","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:11:33.969465Z","iopub.execute_input":"2021-06-10T15:11:33.969825Z","iopub.status.idle":"2021-06-10T15:11:33.973842Z","shell.execute_reply.started":"2021-06-10T15:11:33.969788Z","shell.execute_reply":"2021-06-10T15:11:33.972967Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Chia dữ liệu để huấn luyện với tỉ lệ 90:10**","metadata":{}},{"cell_type":"code","source":"# split to train and val\ntrain, val = train_test_split(train, test_size=0.1, random_state=2018) ","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:11:33.975236Z","iopub.execute_input":"2021-06-10T15:11:33.975606Z","iopub.status.idle":"2021-06-10T15:11:34.532855Z","shell.execute_reply.started":"2021-06-10T15:11:33.975571Z","shell.execute_reply":"2021-06-10T15:11:34.531992Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fill up the missing values\ntrain_X = train[\"question_text\"].fillna(\"_na_\").values\nval_X = val[\"question_text\"].fillna(\"_na_\").values\ntest_X = test[\"question_text\"].fillna(\"_##_\").values","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:11:34.53411Z","iopub.execute_input":"2021-06-10T15:11:34.534436Z","iopub.status.idle":"2021-06-10T15:11:34.814949Z","shell.execute_reply.started":"2021-06-10T15:11:34.534403Z","shell.execute_reply":"2021-06-10T15:11:34.814055Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Get the target values\ntrain_y = train['target'].values\nval_y = val['target'].values ","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:11:34.818382Z","iopub.execute_input":"2021-06-10T15:11:34.818648Z","iopub.status.idle":"2021-06-10T15:11:34.823995Z","shell.execute_reply.started":"2021-06-10T15:11:34.818622Z","shell.execute_reply":"2021-06-10T15:11:34.823254Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Tokenize**\n\nTokenize các câu hỏi thành các token(các từ) sau đó gán chỉ số cho từng từ và thay từng từ thành chỉ số tương ứng của từ đó.","metadata":{}},{"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X)) # gán chỉ số cho từng từ\nword_index = tokenizer.word_index # lấy chỉ số của các từ\ntrain_X = tokenizer.texts_to_sequences(train_X) # thay thế các từ bởi chỉ số tương ứng của từ đó\nval_X = tokenizer.texts_to_sequences(val_X)  # thay thế các từ bởi chỉ số tương ứng của từ đó\ntest_X = tokenizer.texts_to_sequences(test_X) # thay thế các từ bởi chỉ số tương ứng của từ đó","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:11:34.825603Z","iopub.execute_input":"2021-06-10T15:11:34.82603Z","iopub.status.idle":"2021-06-10T15:12:22.786385Z","shell.execute_reply.started":"2021-06-10T15:11:34.825994Z","shell.execute_reply":"2021-06-10T15:12:22.785489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Pad chuỗi** - nếu số lượng từ trong câu hỏi lớn hơn 'max_len' thì chuyển thành 'max_len' hoặc nếu số từ trong văn bản ít hơn 'max_len' thì bổ sung thêm số 0 vào các giá trị còn lại","metadata":{}},{"cell_type":"code","source":"train_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:12:22.787682Z","iopub.execute_input":"2021-06-10T15:12:22.788034Z","iopub.status.idle":"2021-06-10T15:12:34.999373Z","shell.execute_reply.started":"2021-06-10T15:12:22.788Z","shell.execute_reply":"2021-06-10T15:12:34.998532Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Tráo dữ liệu**","metadata":{}},{"cell_type":"code","source":"#shuffling the data\nnp.random.seed(1203)\ntrn_idx = np.random.permutation(len(train_X))\nval_idx = np.random.permutation(len(val_X))\n\ntrain_X = train_X[trn_idx]\nval_X = val_X[val_idx]\ntrain_y = train_y[trn_idx]\nval_y = val_y[val_idx]   ","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:12:35.001049Z","iopub.execute_input":"2021-06-10T15:12:35.001378Z","iopub.status.idle":"2021-06-10T15:12:35.244656Z","shell.execute_reply.started":"2021-06-10T15:12:35.001344Z","shell.execute_reply":"2021-06-10T15:12:35.243803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Nếu phân lớp không sử dụng từ điển thì được score là 0.63:\nhttps://www.kaggle.com/tuaaanminh/quora-insincere-classification-without-embeddings\n\nỞ notebook này, để tăng hiệu quả phân lớp cũng như tăng điểm score, ta sẽ sử dụng từ điển chứa trọng số của các từ đã được huấn luyện sẵn (Glove)","metadata":{}},{"cell_type":"code","source":"! unzip ../input/quora-insincere-questions-classification/embeddings.zip -d input/","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:12:35.246143Z","iopub.execute_input":"2021-06-10T15:12:35.246501Z","iopub.status.idle":"2021-06-10T15:16:10.717578Z","shell.execute_reply.started":"2021-06-10T15:12:35.246462Z","shell.execute_reply":"2021-06-10T15:16:10.716632Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Ta sẽ tạo ma trận embedding chỉ chứa những từ trong ma trận word2vec mà có trong word_index","metadata":{}},{"cell_type":"code","source":"EMBEDDING_FILE = './input/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE)) # lấy hệ số của từng từ trong tập embedding\n\nall_embs = np.stack(embeddings_index.values()) \nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\n# word_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size)) # tạo hệ số ma trận ngẫu nhiên \nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word) \n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector # thay các hệ số của những từ có trong tập embedding","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:16:10.719041Z","iopub.execute_input":"2021-06-10T15:16:10.719304Z","iopub.status.idle":"2021-06-10T15:20:25.8489Z","shell.execute_reply.started":"2021-06-10T15:16:10.719276Z","shell.execute_reply":"2021-06-10T15:20:25.828937Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# MODELING","metadata":{}},{"cell_type":"markdown","source":"**Định nghĩa F1-score**\n\nDo dữ liệu bị mất cân bằng nên metric phù hợp nhất là F1 score","metadata":{}},{"cell_type":"code","source":"!pip install tensorflow-addons","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:32:05.009366Z","iopub.execute_input":"2021-06-10T15:32:05.009724Z","iopub.status.idle":"2021-06-10T15:32:32.096121Z","shell.execute_reply.started":"2021-06-10T15:32:05.009673Z","shell.execute_reply":"2021-06-10T15:32:32.095111Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1 = tfa.metrics.F1Score(num_classes=1, threshold=0.5)\naccuracy = tf.keras.metrics.BinaryAccuracy(\n    name='binary_accuracy', dtype=None, threshold=0.5\n)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:33:04.157622Z","iopub.execute_input":"2021-06-10T15:33:04.157984Z","iopub.status.idle":"2021-06-10T15:33:04.179639Z","shell.execute_reply.started":"2021-06-10T15:33:04.157953Z","shell.execute_reply":"2021-06-10T15:33:04.17882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Định nghĩa hàm Learning rate và Early stopping để giúp hàm loss hội tụ hiệu quả hơn**","metadata":{}},{"cell_type":"code","source":"reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=2, min_lr=0.0001, verbose=0)\nearlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=5, verbose=1, mode='auto', restore_best_weights=True)\ncallbacks = [reduce_lr, earlystopping]","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:33:08.022837Z","iopub.execute_input":"2021-06-10T15:33:08.023154Z","iopub.status.idle":"2021-06-10T15:33:08.029681Z","shell.execute_reply.started":"2021-06-10T15:33:08.023126Z","shell.execute_reply":"2021-06-10T15:33:08.028413Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Thiết lập model**","metadata":{}},{"cell_type":"markdown","source":"Ta sử dụng LSTM - là một dạng đặc biệt của RNN, nó có khả năng học được các phụ thuộc xa.\nDo đó, mô hình LSTM phù hợp với dữ liệu text vì có thể học được thông tin dạng chuỗi.","metadata":{}},{"cell_type":"code","source":"def model_lstm_atten(embedding_matrix):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(LSTM(128, return_sequences=True))(x)\n    x = Bidirectional(LSTM(64, return_sequences=True))(x)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(64, activation=\"relu\")(x)\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=[f1])\n    return model","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:33:45.742848Z","iopub.execute_input":"2021-06-10T15:33:45.743171Z","iopub.status.idle":"2021-06-10T15:33:45.749857Z","shell.execute_reply.started":"2021-06-10T15:33:45.743141Z","shell.execute_reply":"2021-06-10T15:33:45.748849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = model_lstm_atten(embedding_matrix)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:33:46.331673Z","iopub.execute_input":"2021-06-10T15:33:46.332033Z","iopub.status.idle":"2021-06-10T15:33:48.016974Z","shell.execute_reply.started":"2021-06-10T15:33:46.332002Z","shell.execute_reply":"2021-06-10T15:33:48.015881Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Train model**","metadata":{}},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=10, validation_data=(val_X, val_y), callbacks=callbacks)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T15:34:02.175629Z","iopub.execute_input":"2021-06-10T15:34:02.175987Z","iopub.status.idle":"2021-06-10T15:55:50.412044Z","shell.execute_reply.started":"2021-06-10T15:34:02.175957Z","shell.execute_reply":"2021-06-10T15:55:50.411245Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Predict**","metadata":{}},{"cell_type":"code","source":"pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)\npred_test_y = model.predict([test_X], batch_size=1024, verbose=0)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T16:01:19.184641Z","iopub.execute_input":"2021-06-10T16:01:19.185012Z","iopub.status.idle":"2021-06-10T16:01:46.950221Z","shell.execute_reply.started":"2021-06-10T16:01:19.184982Z","shell.execute_reply":"2021-06-10T16:01:46.949295Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Xác định threshold có F1 cao nhất với threshold trong khoảng từ 0.10 đến 0.50**","metadata":{}},{"cell_type":"code","source":"def f1_smart(y_true, y_pred):\n    thresholds = []\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        res = f1_score(y_true, (y_pred > thresh).astype(int))\n        thresholds.append([thresh, res])\n        print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n\n    thresholds.sort(key=lambda x: x[1], reverse=True)\n    best_thresh = thresholds[0][0]\n    best_f1 = thresholds[0][1]\n    print(\"Best threshold: \", best_thresh)\n    return  best_f1, best_thresh","metadata":{"execution":{"iopub.status.busy":"2021-06-10T16:01:46.9517Z","iopub.execute_input":"2021-06-10T16:01:46.952068Z","iopub.status.idle":"2021-06-10T16:01:46.959716Z","shell.execute_reply.started":"2021-06-10T16:01:46.952029Z","shell.execute_reply":"2021-06-10T16:01:46.958808Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1, threshold = f1_smart(val_y, pred_val_y)\nprint('Optimal F1: {} at threshold: {}'.format(f1, threshold))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T16:01:46.961741Z","iopub.execute_input":"2021-06-10T16:01:46.962365Z","iopub.status.idle":"2021-06-10T16:01:48.173212Z","shell.execute_reply.started":"2021-06-10T16:01:46.962223Z","shell.execute_reply":"2021-06-10T16:01:48.172484Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# INFERENCE","metadata":{}},{"cell_type":"code","source":"pred_test_y = (pred_test_y >threshold).astype(int)\nout_df = pd.DataFrame({\"qid\":test[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T16:04:58.174013Z","iopub.execute_input":"2021-06-10T16:04:58.17443Z","iopub.status.idle":"2021-06-10T16:04:59.278343Z","shell.execute_reply.started":"2021-06-10T16:04:58.174394Z","shell.execute_reply":"2021-06-10T16:04:59.277505Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# REFERENCE\nhttps://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings\n\nhttps://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings\n","metadata":{}},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}