{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"#**Quora Insincere Questions Classification**\n\n\n\n","metadata":{"id":"kTDEQ4yPS2rI"}},{"cell_type":"code","source":"# from google.colab import drive\n# drive.mount('/content/gdrive')\n# root_path = 'gdrive/MyDrive/ML project/'","metadata":{"id":"1EuQSx2oSEod","outputId":"e847fca7-bda2-4b0b-d552-de4fd92108d6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"oZXgMp_mJs9_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import pandas as pd \nimport numpy as np \n%matplotlib inline\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom gensim.models import KeyedVectors\nimport operator\nimport gc\nimport matplotlib as mp \nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom sklearn import metrics\nimport re\n","metadata":{"id":"iIYht7mQoPQE"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Dữ liệu","metadata":{"id":"3GcnFFpqPvPP"}},{"cell_type":"code","source":"data_raw = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ndata_raw.head()","metadata":{"id":"WlsU-OR1oYfb","outputId":"2c67b7a2-7d9e-4dab-aa86-2b689fdb5a66"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data=pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")\n","metadata":{"id":"ueL0Ihv1-c-9"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_raw.info()","metadata":{"id":"fOG3PfZBpQoL","outputId":"dab10a13-0fae-4157-ffb9-65c6b93d6a5e"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_raw.target.value_counts()","metadata":{"id":"EDf0wtCqpW5k","outputId":"8b46c286-eae2-47a4-8b72-abfccf904496"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(data=data_raw, x='target')","metadata":{"id":"HkunWfPcpbDe","outputId":"5f2020b1-573d-44fd-8562-1009f47aba0d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"data_raw.target.value_counts(normalize=True)","metadata":{"id":"tKy2CQ4OJuj3","outputId":"9e34d2bf-635b-4973-f1f5-6545689c486f"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_questions = data_raw[data_raw['target'] == 1].question_text\nsincere_questions = data_raw[data_raw['target'] == 0].question_text","metadata":{"id":"dhj0UJY6LHF5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_questions.sample(n=3, random_state=0).values","metadata":{"id":"rrBaYhRZO4Xe","outputId":"26dc900e-a529-4beb-efad-018eec7ecc31"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train,val=train_test_split(data_raw,test_size=0.004,stratify=data_raw.target,random_state=123)\nprint(\"Shape of the Training set :\",train.shape)\nprint(\"Shape of the Validation set :\",val.shape)","metadata":{"id":"4tWC4qeTPFas","outputId":"70eba0d1-b501-4c53-de0a-09ac4c9bf802"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Tiền xử lý","metadata":{"id":"uwPdwF6-P_yC"}},{"cell_type":"code","source":"import nltk\nimport sys\nimport spacy\n\n# nltk.download('stopwords')\n# nltk.download('averaged_perceptron_tagger')\n# nltk.download('wordnet')\n\n# from nltk.corpus import stopwords\nimport string\n","metadata":{"id":"915NLS5X_2sP"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# nltk_stopword = stopwords.words('english')","metadata":{"id":"qyQRqCHNiGST"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"contractions={\"I'm\": 'I am',\n \"I'm'a\": 'I am about to',\n \"I'm'o\": 'I am going to',\n \"I've\": 'I have',\n \"I'll\": 'I will',\n \"I'll've\": 'I will have',\n \"I'd\": 'I would',\n \"I'd've\": 'I would have',\n 'Whatcha': 'What are you',\n \"amn't\": 'am not',\n \"ain't\": 'are not',\n \"aren't\": 'are not',\n \"'cause\": 'because',\n \"can't\": 'can not',\n \"can't've\": 'can not have',\n \"could've\": 'could have',\n \"couldn't\": 'could not',\n \"couldn't've\": 'could not have',\n \"daren't\": 'dare not',\n \"daresn't\": 'dare not',\n \"dasn't\": 'dare not',\n \"didn't\": 'did not',\n 'didn’t': 'did not',\n \"don't\": 'do not',\n 'don’t': 'do not',\n \"doesn't\": 'does not',\n \"e'er\": 'ever',\n \"everyone's\": 'everyone is',\n 'finna': 'fixing to',\n 'gimme': 'give me',\n \"gon't\": 'go not',\n 'gonna': 'going to',\n 'gotta': 'got to',\n \"hadn't\": 'had not',\n \"hadn't've\": 'had not have',\n \"hasn't\": 'has not',\n \"haven't\": 'have not',\n \"he've\": 'he have',\n \"he's\": 'he is',\n \"he'll\": 'he will',\n \"he'll've\": 'he will have',\n \"he'd\": 'he would',\n \"he'd've\": 'he would have',\n \"here's\": 'here is',\n \"how're\": 'how are',\n \"how'd\": 'how did',\n \"how'd'y\": 'how do you',\n \"how's\": 'how is',\n \"how'll\": 'how will',\n \"isn't\": 'is not',\n \"it's\": 'it is',\n \"'tis\": 'it is',\n \"'twas\": 'it was',\n \"it'll\": 'it will',\n \"it'll've\": 'it will have',\n \"it'd\": 'it would',\n \"it'd've\": 'it would have',\n 'kinda': 'kind of',\n \"let's\": 'let us',\n 'luv': 'love',\n \"ma'am\": 'madam',\n \"may've\": 'may have',\n \"mayn't\": 'may not',\n \"might've\": 'might have',\n \"mightn't\": 'might not',\n \"mightn't've\": 'might not have',\n \"must've\": 'must have',\n \"mustn't\": 'must not',\n \"mustn't've\": 'must not have',\n \"needn't\": 'need not',\n \"needn't've\": 'need not have',\n \"ne'er\": 'never',\n \"o'\": 'of',\n \"o'clock\": 'of the clock',\n \"ol'\": 'old',\n \"oughtn't\": 'ought not',\n \"oughtn't've\": 'ought not have',\n \"o'er\": 'over',\n \"shan't\": 'shall not',\n \"sha'n't\": 'shall not',\n \"shalln't\": 'shall not',\n \"shan't've\": 'shall not have',\n \"she's\": 'she is',\n \"she'll\": 'she will',\n \"she'd\": 'she would',\n \"she'd've\": 'she would have',\n \"should've\": 'should have',\n \"shouldn't\": 'should not',\n \"shouldn't've\": 'should not have',\n \"so've\": 'so have',\n \"so's\": 'so is',\n \"somebody's\": 'somebody is',\n \"someone's\": 'someone is',\n \"something's\": 'something is',\n 'sux': 'sucks',\n \"that're\": 'that are',\n \"that's\": 'that is',\n \"that'll\": 'that will',\n \"that'd\": 'that would',\n \"that'd've\": 'that would have',\n 'em': 'them',\n \"there're\": 'there are',\n \"there's\": 'there is',\n \"there'll\": 'there will',\n \"there'd\": 'there would',\n \"there'd've\": 'there would have',\n \"these're\": 'these are',\n \"they're\": 'they are',\n \"they've\": 'they have',\n \"they'll\": 'they will',\n \"they'll've\": 'they will have',\n \"they'd\": 'they would',\n \"they'd've\": 'they would have',\n \"this's\": 'this is',\n \"those're\": 'those are',\n \"to've\": 'to have',\n 'wanna': 'want to',\n \"wasn't\": 'was not',\n \"we're\": 'we are',\n \"we've\": 'we have',\n \"we'll\": 'we will',\n \"we'll've\": 'we will have',\n \"we'd\": 'we would',\n \"we'd've\": 'we would have',\n \"weren't\": 'were not',\n \"what're\": 'what are',\n \"what'd\": 'what did',\n \"what've\": 'what have',\n \"what's\": 'what is',\n \"what'll\": 'what will',\n \"what'll've\": 'what will have',\n \"when've\": 'when have',\n \"when's\": 'when is',\n \"where're\": 'where are',\n \"where'd\": 'where did',\n \"where've\": 'where have',\n \"where's\": 'where is',\n \"which's\": 'which is',\n \"who're\": 'who are',\n \"who've\": 'who have',\n \"who's\": 'who is',\n \"who'll\": 'who will',\n \"who'll've\": 'who will have',\n \"who'd\": 'who would',\n \"who'd've\": 'who would have',\n \"why're\": 'why are',\n \"why'd\": 'why did',\n \"why've\": 'why have',\n \"why's\": 'why is',\n \"will've\": 'will have',\n \"won't\": 'will not',\n \"won't've\": 'will not have',\n \"would've\": 'would have',\n \"wouldn't\": 'would not',\n \"wouldn't've\": 'would not have',\n \"y'all\": 'you all',\n \"y'all're\": 'you all are',\n \"y'all've\": 'you all have',\n \"y'all'd\": 'you all would',\n \"y'all'd've\": 'you all would have',\n \"you're\": 'you are',\n \"you've\": 'you have',\n \"you'll've\": 'you shall have',\n \"you'll\": 'you will',\n \"you'd\": 'you would',\n \"you'd've\": 'you would have',\n 'jan.': 'january',\n 'feb.': 'february',\n 'mar.': 'march',\n 'apr.': 'april',\n 'jun.': 'june',\n 'jul.': 'july',\n 'aug.': 'august',\n 'sep.': 'september',\n 'oct.': 'october',\n 'nov.': 'november',\n 'dec.': 'december',\n 'I’m': 'I am',\n 'I’m’a': 'I am about to',\n 'I’m’o': 'I am going to',\n 'I’ve': 'I have',\n 'I’ll': 'I will',\n 'I’ll’ve': 'I will have',\n 'I’d': 'I would',\n 'I’d’ve': 'I would have',\n 'amn’t': 'am not',\n 'ain’t': 'are not',\n 'aren’t': 'are not',\n '’cause': 'because',\n 'can’t': 'can not',\n 'can’t’ve': 'can not have',\n 'could’ve': 'could have',\n 'couldn’t': 'could not',\n 'couldn’t’ve': 'could not have',\n 'daren’t': 'dare not',\n 'daresn’t': 'dare not',\n 'dasn’t': 'dare not',\n 'doesn’t': 'does not',\n 'e’er': 'ever',\n 'everyone’s': 'everyone is',\n 'gon’t': 'go not',\n 'hadn’t': 'had not',\n 'hadn’t’ve': 'had not have',\n 'hasn’t': 'has not',\n 'haven’t': 'have not',\n 'he’ve': 'he have',\n 'he’s': 'he is',\n 'he’ll': 'he will',\n 'he’ll’ve': 'he will have',\n 'he’d': 'he would',\n 'he’d’ve': 'he would have',\n 'here’s': 'here is',\n 'how’re': 'how are',\n 'how’d': 'how did',\n 'how’d’y': 'how do you',\n 'how’s': 'how is',\n 'how’ll': 'how will',\n 'isn’t': 'is not',\n 'it’s': 'it is',\n '’tis': 'it is',\n '’twas': 'it was',\n 'it’ll': 'it will',\n 'it’ll’ve': 'it will have',\n 'it’d': 'it would',\n 'it’d’ve': 'it would have',\n 'let’s': 'let us',\n 'ma’am': 'madam',\n 'may’ve': 'may have',\n 'mayn’t': 'may not',\n 'might’ve': 'might have',\n 'mightn’t': 'might not',\n 'mightn’t’ve': 'might not have',\n 'must’ve': 'must have',\n 'mustn’t': 'must not',\n 'mustn’t’ve': 'must not have',\n 'needn’t': 'need not',\n 'needn’t’ve': 'need not have',\n 'ne’er': 'never',\n 'o’': 'of',\n 'o’clock': 'of the clock',\n 'ol’': 'old',\n 'oughtn’t': 'ought not',\n 'oughtn’t’ve': 'ought not have',\n 'o’er': 'over',\n 'shan’t': 'shall not',\n 'sha’n’t': 'shall not',\n 'shalln’t': 'shall not',\n 'shan’t’ve': 'shall not have',\n 'she’s': 'she is',\n 'she’ll': 'she will',\n 'she’d': 'she would',\n 'she’d’ve': 'she would have',\n 'should’ve': 'should have',\n 'shouldn’t': 'should not',\n 'shouldn’t’ve': 'should not have',\n 'so’ve': 'so have',\n 'so’s': 'so is',\n 'somebody’s': 'somebody is',\n 'someone’s': 'someone is',\n 'something’s': 'something is',\n 'that’re': 'that are',\n 'that’s': 'that is',\n 'that’ll': 'that will',\n 'that’d': 'that would',\n 'that’d’ve': 'that would have',\n 'there’re': 'there are',\n 'there’s': 'there is',\n 'there’ll': 'there will',\n 'there’d': 'there would',\n 'there’d’ve': 'there would have',\n 'these’re': 'these are',\n 'they’re': 'they are',\n 'they’ve': 'they have',\n 'they’ll': 'they will',\n 'they’ll’ve': 'they will have',\n 'they’d': 'they would',\n 'they’d’ve': 'they would have',\n 'this’s': 'this is',\n 'those’re': 'those are',\n 'to’ve': 'to have',\n 'wasn’t': 'was not',\n 'we’re': 'we are',\n 'we’ve': 'we have',\n 'we’ll': 'we will',\n 'we’ll’ve': 'we will have',\n 'we’d': 'we would',\n 'we’d’ve': 'we would have',\n 'weren’t': 'were not',\n 'what’re': 'what are',\n 'what’d': 'what did',\n 'what’ve': 'what have',\n 'what’s': 'what is',\n 'what’ll': 'what will',\n 'what’ll’ve': 'what will have',\n 'when’ve': 'when have',\n 'when’s': 'when is',\n 'where’re': 'where are',\n 'where’d': 'where did',\n 'where’ve': 'where have',\n 'where’s': 'where is',\n 'which’s': 'which is',\n 'who’re': 'who are',\n 'who’ve': 'who have',\n 'who’s': 'who is',\n 'who’ll': 'who will',\n 'who’ll’ve': 'who will have',\n 'who’d': 'who would',\n 'who’d’ve': 'who would have',\n 'why’re': 'why are',\n 'why’d': 'why did',\n 'why’ve': 'why have',\n 'why’s': 'why is',\n 'will’ve': 'will have',\n 'won’t': 'will not',\n 'won’t’ve': 'will not have',\n 'would’ve': 'would have',\n 'wouldn’t': 'would not',\n 'wouldn’t’ve': 'would not have',\n 'y’all': 'you all',\n 'y’all’re': 'you all are',\n 'y’all’ve': 'you all have',\n 'y’all’d': 'you all would',\n 'y’all’d’ve': 'you all would have',\n 'you’re': 'you are',\n 'you’ve': 'you have',\n 'you’ll’ve': 'you shall have',\n 'you’ll': 'you will',\n 'you’d': 'you would',\n 'you’d’ve': 'you would have'}\n\ndef contraction_fix(word):\n    try:\n        a=contractions[word]\n    except KeyError:\n        a=word\n    return a","metadata":{"id":"lhv4HgTwfkoF"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Preprocess(doc):\n    corpus=[]\n    for text in tqdm(doc):\n        text=\" \".join([contraction_fix(w) for w in text.split()])\n        text = text.lower()\n        text=re.sub(r'[^a-z0-9A-Z]',\" \",text)\n        text=re.sub(r'[0-9]{1}',\"#\",text)\n        text=re.sub(r'[0-9]{2}','##',text)\n        text=re.sub(r'[0-9]{3}','###',text)\n        text=re.sub(r'[0-9]{4}','####',text)\n        text=re.sub(r'[0-9]{5,}','#####',text)\n        corpus.append(text)\n    return corpus\ndef get_vocab(corpus):\n    vocab={}\n    for text in tqdm(corpus):\n        for word in text.split():\n            try:\n                vocab[word]+=1\n            except KeyError:\n                vocab[word]=1\n    vocab=dict(sorted(vocab.items(),reverse=True ,key=lambda item: item[1]))\n    return vocab","metadata":{"id":"CVBDTZWc3JKT"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> Mô hình sẽ học tốt hơn nếu như các từ có trọng số hợp lý, vì vậy ta sẽ dùng 1 từ điển chứa trọng số của các từ đã được huấn luyện sẵn","metadata":{"id":"JW3bhHltT4M5"}},{"cell_type":"markdown","source":"## Sử dụng Fasttext Embedding\nMỗi từ trong dữ liệu Fasttext được biểu diễn bằng 300 chiều vector đã có trọng số hợp lý (đã được huấn luyện sẵn), vì vậy ta không cần train lại các trọng số đó.\n","metadata":{"id":"IcRNzQszSlHD"}},{"cell_type":"markdown","source":"tải dữ liệu pretrain vocab","metadata":{"id":"G_XVuL8JTerh"}},{"cell_type":"code","source":"!unzip ../input/quora-insincere-questions-classification/embeddings.zip\n","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nmodel_embed = KeyedVectors.load_word2vec_format('./wiki-news-300d-1M/wiki-news-300d-1M.vec')","metadata":{"id":"SqwzieBJwbGm"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> ta cần kiểm tra độ bao phủ của dữ liệu từ trong ptetrain Fasttext xem dữ liệu có bao phủ tốt so với dữ liệu của bài toán không.","metadata":{"id":"4PSWS_UjUEWt"}},{"cell_type":"code","source":"### kiểm tra xem độ bao phủ của từ điển nhúng với từ điển của dữ liệu\ndef check_voc(vocab,model):\n    embed_words=[]\n    out_vocab={}\n    total_words=0\n    total_text=0\n    for i in tqdm(vocab):\n        try:\n            vec=model[i]\n            embed_words.append(vec) # list các vector của những từ xuất hiện cả embed vocab và vocab dữ liệu\n            total_words+=vocab[i] # tổng số từ xuất hiện trong dữ liệu mà có trong cả embed vocab, tính cả từ giống nhau\n        except KeyError: ## những từ không xuất hiện trong embed model\n            out_vocab[i]=vocab[i]\n            total_text+=vocab[i] # total text bằng chính tổng lượng từ trong dữ liệu\n    print(\"The {:.2f}% of vocabularies have Covered of corpus\".format(100*len(embed_words)/len(vocab)))\n    print(\"The {:.2f}% of total text had coverded \".format((100*total_words/(total_words+total_text))))\n    return out_vocab","metadata":{"id":"ZAtOdar7DUeI"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"gọi hàm kiểm tra:","metadata":{"id":"AMokrIl5Vg0R"}},{"cell_type":"code","source":"total_text=pd.concat([data_raw.question_text,test_data.question_text])\nvocabulary=get_vocab(total_text)\noov=check_voc(vocabulary,model_embed)","metadata":{"id":"D_sqVGEn4KIt","outputId":"e1db7948-93bb-46b5-b809-9687f14d1c81"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> đây là một tỷ lệ chưa được tốt lắm, ta cần preprocess để loại bỏ các kí tự thừa, khi đó tỷ lệ sẽ cao hơn.","metadata":{"id":"Gq-cyR6Xbkh_"}},{"cell_type":"markdown","source":"xem những từ không xuất hiện trong model embed và tần suất xuất hiện trong dữ liệu:","metadata":{"id":"UypCblLmVogZ"}},{"cell_type":"code","source":"sort_oov=dict(sorted(oov.items(), key=operator.itemgetter(1),reverse=True))\ndict(list(sort_oov.items())[:50])","metadata":{"id":"Iwg0gV9f4m1Z","outputId":"8b56159e-5422-4200-9c25-6828703d2373"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> các từ đi liền với dấu hỏi sẽ khiến cho mô hình embed không nhận ra được từ, vì vậy cần phải loại bỏ các kí tự.","metadata":{"id":"Ha0weSE8b53n"}},{"cell_type":"markdown","source":"Tiền xử lý để độ bao phủ cao hơn:","metadata":{"id":"sNSQ9d2iZw4P"}},{"cell_type":"code","source":"pre_text=Preprocess(total_text)\nvocabulary=get_vocab(pre_text)\noov=check_voc(vocabulary,model_embed)","metadata":{"id":"oGSNe9OG5cU9","outputId":"959115db-3e8f-4a77-936b-c0014b8add9e"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"xem những từ không xuất hiện trong model embed và tần suất xuất hiện trong dữ liệu:","metadata":{"id":"lcTrU5dAcLJa"}},{"cell_type":"code","source":"sort_oov=dict(sorted(oov.items(), key=operator.itemgetter(1),reverse=True))\ndict(list(sort_oov.items())[:50])","metadata":{"id":"5-iYnCjo5sz0","outputId":"5ee51733-c68f-410c-fbbc-718e8ab83bc1"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"> có thể thấy những từ không xuất hiện trong model embed giờ là những từ không mang nhiều ý nghia","metadata":{"id":"YU3cv0eYcVel"}},{"cell_type":"code","source":"!pip install tensorflow-addons","metadata":{"id":"11q9oSwvUZIP","outputId":"33fb11c4-064f-44a7-9319-c9409fa08853"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### importing the libraire\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D,GRU\nfrom keras.layers import *\n\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model,load_model\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils import plot_model\nfrom keras.models import Sequential\nfrom keras.callbacks import ModelCheckpoint,EarlyStopping\nfrom keras.optimizers import Adam\nimport tensorflow as tf\nimport tensorflow_addons as tfa\n","metadata":{"id":"OJlv0nQdG3g1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### defining the parameter\nmax_feat=30000\nmax_len=55\nfeat_vec=300","metadata":{"id":"Do5M1Xkx_EKL"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### gettting the index for   each word in vocabulary\ndef get_word_index(vocab):\n    word_index=dict((w,i+1) for i,w in enumerate(vocab.keys()))\n    return word_index\ndef fit_one_hot(word_index,corpus):\n    sent=[]\n    for text in tqdm(corpus):\n        li=[]\n        for word in text.split():\n            try:\n                li.append(word_index[word])\n            except KeyError:\n                li.append(0)\n        sent.append(li)\n    return sent","metadata":{"id":"bX6HbI4oEmxv"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"id":"mxmmXzRrGHww"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(vocabulary)","metadata":{"id":"ygwd3T-t4MRH","outputId":"c4a9d067-6355-4ca2-9412-523d944b59a5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\n### preprocessing the text\nword_index=get_word_index(vocabulary)\n### preprocess the data\ntrain_text=Preprocess(train.question_text)\nval_text=Preprocess(val.question_text)\ntest_text=Preprocess(test_data.question_text)\n\n### encoding the training set\nencoded_docs=fit_one_hot(word_index,train_text)\npadded_doc=pad_sequences(encoded_docs,maxlen=max_len,padding=\"post\")\n\n### encoding the Validation set\nencoded_docs=fit_one_hot(word_index,val_text)\nval_padded_doc=pad_sequences(encoded_docs,maxlen=max_len,padding=\"post\")\n","metadata":{"id":"qozOHKQs_E62","outputId":"dcb741f8-b800-4af6-8903-2197b20cf90d"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"### encoding the Test set\nencoded_docs=fit_one_hot(word_index,test_text)\ntest_padded_doc=pad_sequences(encoded_docs,maxlen=max_len,padding=\"post\")","metadata":{"id":"LhHOnE1TQ5GP","outputId":"c290de97-4b0e-49f2-b359-9c3cb57ff9ec"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_padded_doc[0]","metadata":{"id":"lNa2YbW6RKSj","outputId":"4acebe4d-d315-4b74-bc65-cf2d0d1376df"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"padded_doc[0]","metadata":{"id":"3JNxLyv1HAS-","outputId":"753af371-c1a1-413a-ec99-47d1d1bdeebb"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## dựng model","metadata":{"id":"wAKPpuPFdHxa"}},{"cell_type":"code","source":"count=0\nembedding_mat=np.zeros((len(vocabulary)+1,300)) # tạo ma trận weight của từ dựa trên từ điển embed\nfor word,i in tqdm(word_index.items()):\n    try:\n        vec=model_embed[word]\n        embedding_mat[i]=vec\n    except KeyError:\n        count+=1\n        continue\n\nprint(\"Number of Out of Vocabulary\",count)","metadata":{"id":"9PhnlI1WCLap","outputId":"96cb8f05-fe99-4399-ed53-71b956da039a"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"f1 = tfa.metrics.F1Score(num_classes=1, threshold=0.5)\naccuracy = tf.keras.metrics.BinaryAccuracy(\n    name='binary_accuracy', dtype=None, threshold=0.5\n)","metadata":{"id":"xI84c_LSbFjd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"inp = Input(shape=(max_len,))\nx = Embedding(len(vocabulary)+1, feat_vec, weights=[embedding_mat], input_length=max_len,trainable=False)(inp) # trọng số của từ sẽ không được train lại\nx = Bidirectional(CuDNNGRU(128, return_sequences=True))(x)\nx = Conv1D(64,3,activation=\"relu\")(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(128, activation=\"relu\")(x)\nx = Dropout(0.2)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\n\n\nprint(model.summary())","metadata":{"id":"w9FpXsff6mBR","outputId":"e6c50c59-9d0e-4eb6-dceb-6e00752dc2a4"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"nếu chạy nhiều epoch mà để learning rate cố định thì loss sẽ khó hội tụ, do đó cần định nghĩa hàm Reduce Learning rate vè Early stopping.","metadata":{"id":"aACXzx-lco39"}},{"cell_type":"code","source":"### defining some callbacks\nopt=Adam(learning_rate=0.0001)\nbin_loss=tf.keras.losses.BinaryCrossentropy(\n                                            from_logits=False, \n                                            label_smoothing=0,\n                                            name='binary_crossentropy'\n                                        )\n\n## defining the call backs\nearly_stopping=tf.keras.callbacks.EarlyStopping(\n                                                monitor=\"val_loss\",\n                                                patience=5,\n                                                verbose=1,\n                                                mode=\"min\",\n                                                restore_best_weights=True\n                                              )\n### Now reducing the learning rate when the model is not improvinig \nreduce_lr=tf.keras.callbacks.ReduceLROnPlateau(\n                                                monitor=\"val_loss\",\n                                                factor=0.01,\n                                                patience=2,\n                                                verbose=1,\n                                                mode=\"auto\"\n                                            )\n\nmy_callbacks=[early_stopping,reduce_lr]","metadata":{"id":"PabkpipdRDHp"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"chạy model\n","metadata":{"id":"USsCk6_oohk2"}},{"cell_type":"code","source":"model.compile(loss=bin_loss, optimizer=opt, metrics=[f1, accuracy])\n\nhistory = model.fit(padded_doc, train.target, batch_size=512, epochs=25, validation_data=(val_padded_doc, val.target), callbacks = my_callbacks)","metadata":{"id":"iEh5GYcRT5KF","outputId":"11484c79-a25d-4d0a-f6c9-a1c0d4f632ac"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"fix, ax = plt.subplots(figsize=(20, 6))\npd.DataFrame(history.history)[['loss', 'val_loss']].plot(ax=ax, title='Model Loss Curve')","metadata":{"id":"xw42wFNwImOD","outputId":"35ffa047-3bd0-46da-ed16-fd6989ec53b1"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# model.compile(loss=bin_loss, optimizer=Adam(learning_rate=0.0015), metrics=[f1, accuracy])\n\n# history = model.fit(padded_doc, train.target, batch_size=512, epochs=25, validation_data=(val_padded_doc, val.target), callbacks = my_callbacks)","metadata":{"id":"T1vxRV6nJGAc","outputId":"c3311f3f-7415-421d-af19-63a1069ecabe"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# fix, ax = plt.subplots(figsize=(20, 6))\n# pd.DataFrame(history.history)[['loss', 'val_loss']].plot(ax=ax, title='Model Loss Curve')","metadata":{"id":"kniY0OkJI02U","outputId":"f8edc205-5cc5-419e-c902-d2868550098c"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# !mkdir -p saved_model\n# model.save('saved_model/model_v4') ","metadata":{"id":"aTw8CYSEKSC5"},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"kết quả có vẻ tốt hơn so với trước khi preprocess\n","metadata":{"id":"AANsDo83y5N3"}},{"cell_type":"code","source":"y_pre=model.predict(val_padded_doc)\n","metadata":{"id":"irQQwqNTzNjh"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"thresh=0.36\n\nfor thresh in np.arange(0.1,0.5,0.01):\n    print(\"threshold {0:2.2f} f1 score:{1:2.3f}\".format(thresh,metrics.f1_score(val.target,(y_pre>thresh).astype(int))))","metadata":{"id":"rQUFkUR6VBFq","outputId":"fbbe2223-452b-43f6-8466-f73d635f68a4"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.metrics import classification_report\nprint(classification_report(val.target, (y_pre>thresh).astype(int)))","metadata":{"id":"12BpGiBJoC4O","outputId":"84ab3e92-f8c7-428a-c5cf-4b3d778b47dd"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_test_pre=model.predict(test_padded_doc)\ny_test_pre=(y_test_pre>thresh).astype(int)\n\n## Creating the submission File\nsubmit=pd.DataFrame()\nsubmit[\"qid\"]=test_data.qid\nsubmit[\"prediction\"]=y_test_pre\nsubmit.to_csv(\"submission.csv\",index=False)","metadata":{"id":"yVrbfT3-Swko"},"execution_count":null,"outputs":[]}]}