{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Phân tích dữ liệu","metadata":{}},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport pandas as pd\n\nfrom nltk.corpus import stopwords","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:47:35.743239Z","iopub.execute_input":"2022-01-08T14:47:35.743526Z","iopub.status.idle":"2022-01-08T14:47:35.749646Z","shell.execute_reply.started":"2022-01-08T14:47:35.743497Z","shell.execute_reply":"2022-01-08T14:47:35.748554Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\ntrain_df = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:47:20.347734Z","iopub.execute_input":"2022-01-08T14:47:20.348108Z","iopub.status.idle":"2022-01-08T14:47:25.951126Z","shell.execute_reply.started":"2022-01-08T14:47:20.348075Z","shell.execute_reply":"2022-01-08T14:47:25.950007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Xem dữ liệu tập train**","metadata":{}},{"cell_type":"code","source":"train_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:32:57.537342Z","iopub.execute_input":"2022-01-08T13:32:57.537687Z","iopub.status.idle":"2022-01-08T13:32:57.563227Z","shell.execute_reply.started":"2022-01-08T13:32:57.537646Z","shell.execute_reply":"2022-01-08T13:32:57.561231Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Xem dữ liệu tập test**","metadata":{}},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:32:57.565957Z","iopub.execute_input":"2022-01-08T13:32:57.566292Z","iopub.status.idle":"2022-01-08T13:32:57.578517Z","shell.execute_reply.started":"2022-01-08T13:32:57.566249Z","shell.execute_reply":"2022-01-08T13:32:57.577079Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Kiểm tra số lượng câu hỏi và ratio trong tập train**","metadata":{}},{"cell_type":"code","source":"print('Tổng số câu hỏi', train_df.shape[0])\nprint('Câu hỏi chân thành', train_df[train_df['target'] == 0].shape[0])\nprint('Câu hỏi không chân thành', train_df[train_df['target'] == 1].shape[0])","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:32:57.580759Z","iopub.execute_input":"2022-01-08T13:32:57.581294Z","iopub.status.idle":"2022-01-08T13:32:57.730881Z","shell.execute_reply.started":"2022-01-08T13:32:57.58125Z","shell.execute_reply":"2022-01-08T13:32:57.729717Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Kiếm tra số từ xuất hiện nhiều nhất trong cả insincere và sincere questions**","metadata":{}},{"cell_type":"code","source":"class Vocabulary(object):\n    \n    def __init__(self):\n        self.vocab = {}\n        self.STOPWORDS = set()\n        self.STOPWORDS = set(stopwords.words('english'))\n        \n    def build_vocab(self, lines):\n        for line in lines:\n            for word in line.split(' '):\n                word = word.lower()\n                if (word in self.STOPWORDS):\n                    continue\n                if (word not in self.vocab):\n                    self.vocab[word] = 0\n                self.vocab[word] +=1 ","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:32:57.732615Z","iopub.execute_input":"2022-01-08T13:32:57.732921Z","iopub.status.idle":"2022-01-08T13:32:57.741945Z","shell.execute_reply.started":"2022-01-08T13:32:57.73288Z","shell.execute_reply":"2022-01-08T13:32:57.740458Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sincere_vocab = Vocabulary()\nsincere_vocab.build_vocab(train_df[train_df['target'] == 0]['question_text'])\nsincere_vocabulary = sorted(sincere_vocab.vocab.items(), reverse=True, key=lambda kv: kv[1])\nfor word, count in sincere_vocabulary[:10]:\n    print(word, count)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:32:57.743724Z","iopub.execute_input":"2022-01-08T13:32:57.744562Z","iopub.status.idle":"2022-01-08T13:33:06.512274Z","shell.execute_reply.started":"2022-01-08T13:32:57.744517Z","shell.execute_reply":"2022-01-08T13:33:06.511277Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"insincere_vocab = Vocabulary()\ninsincere_vocab.build_vocab(train_df[train_df['target'] == 1]['question_text'])\ninsincere_vocabulary = sorted(insincere_vocab.vocab.items(), reverse=True, key=lambda kv: kv[1])\nfor word, count in insincere_vocabulary[:10]:\n    print(word, count)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:33:06.517476Z","iopub.execute_input":"2022-01-08T13:33:06.520177Z","iopub.status.idle":"2022-01-08T13:33:07.402301Z","shell.execute_reply.started":"2022-01-08T13:33:06.520131Z","shell.execute_reply":"2022-01-08T13:33:07.401275Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Xem thử vài mẫu câu bị đánh giá không chân thành**","metadata":{}},{"cell_type":"code","source":"print(train_df[train_df['target'] == 1]['question_text'])","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:33:07.405197Z","iopub.execute_input":"2022-01-08T13:33:07.405937Z","iopub.status.idle":"2022-01-08T13:33:07.432967Z","shell.execute_reply.started":"2022-01-08T13:33:07.405889Z","shell.execute_reply":"2022-01-08T13:33:07.431648Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Làm sạch dữ liệu","metadata":{}},{"cell_type":"code","source":"import unidecode\nimport re\nimport nltk\nimport string\nimport codecs\nimport spacy\nfrom nltk.corpus import stopwords\nimport pickle","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:33:07.438073Z","iopub.execute_input":"2022-01-08T13:33:07.438368Z","iopub.status.idle":"2022-01-08T13:33:18.172933Z","shell.execute_reply.started":"2022-01-08T13:33:07.438337Z","shell.execute_reply":"2022-01-08T13:33:18.171973Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Clean punctations** ","metadata":{}},{"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£',\n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\\xa0', '\\t',\n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\\u3000', '\\u202f',\n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',\n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\n# replace puncts\ndef clean_puncts(text):\n    text = str(text)\n    for punct in \"/-'\":\n        text = text.replace(punct, ' ')\n    for punct in puncts:\n        text = text.replace(punct, f' {punct} ')\n    return text","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:33:18.17472Z","iopub.execute_input":"2022-01-08T13:33:18.175448Z","iopub.status.idle":"2022-01-08T13:33:18.189794Z","shell.execute_reply.started":"2022-01-08T13:33:18.175403Z","shell.execute_reply":"2022-01-08T13:33:18.188754Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Clean Mispell**","metadata":{}},{"cell_type":"code","source":"mispell_dict = {\n    'grey': 'gray',\n    'litre': 'liter',\n    'labour': 'labor',\n    'favour': 'favor',\n    'colour': 'color',\n    'centre': 'center',\n    'honours': 'honor',\n    'theatre': 'theater',\n    'realise': 'realize',\n    'defence': 'defense',\n    'licence': 'license',\n    'analyse': 'analyze',\n    'practise': 'practice',\n    'behaviour': 'behavior',\n    'neighbour': 'neighbor',\n    'recognise': 'recognize',\n    'organisation':'organization',\n    'Qoura': 'Quora',\n    'quora': 'Quora',\n    'Quorans': 'Quoran',\n    'infty': 'infinity',\n    'judgement': 'judge',\n    'learnt': 'learn',\n    'modelling': 'model',\n    'cancelled': 'cancel',\n    'travelled': 'travel',\n    'travelling': 'travel',\n    'aluminium': 'alumini',\n    'counselling':'counseling',\n    'cheque': 'bill',\n    'upvote': 'agree',\n    'upvotes': 'agree',\n    'vape': 'cigarette',\n    'jewellery': 'jewell',\n    'Fiverr': 'freelance',\n    'programd': 'program',\n    'programme': 'program',\n    'programr': 'programer',\n    'programrs': 'programer',\n    'WeChat': 'socialmedia',\n    'Snapchat': 'socialmedia',\n    'Redmi': 'cellphone',\n    'Xiaomi': 'cellphone',\n    'OnePlus': 'cellphone',\n    'cryptos': 'crypto',\n    'bitcoin': 'crypto',\n    'Coinbase': 'crypto',\n    'bitcoins': 'crypto',\n    'ethereum': 'crypto',\n    'Ethereum': 'crypto',\n    'Blockchain': 'crypto',\n    'blockchain': 'crypto',\n    'cryptocurrency': 'crypto',\n    'cryptocurrencies': 'crypto',\n    '₹': 'rupee',\n    'Brexit': 'Britain exit',\n    'Paytm': 'Pay Through Mobile',\n    'KVPY': 'Kishore Vaigyanik Protsahan Yojana',\n    'GDPR': 'General Data Protection Regulation',\n    'INTJ': 'Introversion Intuition Thinking Judgment',\n    \"ain't\": \"is not\", \"aren't\": \"are not\", \"can't\": \"cannot\", \"'cause\": \"because\",\n    \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\", \"doesn't\": \"does not\",\n    \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\",\n    \"he'd\": \"he would\", \"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\",\n    \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\", \"i'd\": \"i would\",\n    \"i'd've\": \"i would have\", \"i'll\": \"i will\", \"i'll've\": \"I will have\", \"i'm\": \"i am\",\n    \"i've\": \"I have\", \"isn't\": \"is not\", \"it'd\": \"it would\",\n    \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\", \"it's\": \"it is\",\n    \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\n    \"mightn't\": \"might not\", \"mightn't've\": \"might not have\", \"must've\": \"must have\",\n    \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\",\n    \"needn't've\": \"need not have\", \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\",\n    \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\",\n    \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\",\n    \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\",\n    \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\",\n    \"so've\": \"so have\", \"so's\": \"so as\", \"this's\": \"this is\", \"that'd\": \"that would\",\n    \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\",\n    \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\n    \"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\",\n    \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\",\n    \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\",\n    \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\",\n    \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\",\n    \"what're\": \"what are\", \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\",\n    \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\",\n    \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\",\n    \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\",\n    \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\",\n    \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n    \"y'all'd've\": \"you all would have\", \"y'all're\": \"you all are\", \"y'all've\": \"you all have\",\n    \"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\",\n    \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color',\n    'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',\n    'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',\n    'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize',\n    'youtu ': 'youtube ', 'qoura': 'quora', 'sallary': 'salary', 'whta': 'what',\n    'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can',\n    'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doi': 'do I',\n    'thebest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation',\n    'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis',\n    'etherium': 'ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017',\n    '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess',\n    \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization',\n    'demonitization': 'demonetization', 'demonetisation': 'demonetization'\n}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef clean_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)\n\ndef contraction_fix(word):\n    try:\n        a=mispell_dict[word]\n    except KeyError:\n        a=word\n    return a","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:33:18.191954Z","iopub.execute_input":"2022-01-08T13:33:18.192344Z","iopub.status.idle":"2022-01-08T13:33:18.231557Z","shell.execute_reply.started":"2022-01-08T13:33:18.192301Z","shell.execute_reply":"2022-01-08T13:33:18.230323Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: clean_puncts(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_puncts(x))\n\ntrain_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: clean_misspell(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: clean_misspell(x))","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:33:18.23338Z","iopub.execute_input":"2022-01-08T13:33:18.233768Z","iopub.status.idle":"2022-01-08T13:35:50.088988Z","shell.execute_reply.started":"2022-01-08T13:33:18.233723Z","shell.execute_reply":"2022-01-08T13:35:50.087791Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Sử dụng CountVectorizer / Logistic Regression**","metadata":{}},{"cell_type":"code","source":"'''\nfrom nltk.stem import PorterStemmer\nfrom nltk.tokenize import word_tokenize\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.metrics import f1_score, classification_report\nfrom sklearn.pipeline import Pipeline\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.09089Z","iopub.execute_input":"2022-01-08T13:35:50.091506Z","iopub.status.idle":"2022-01-08T13:35:50.100811Z","shell.execute_reply.started":"2022-01-08T13:35:50.091462Z","shell.execute_reply":"2022-01-08T13:35:50.099618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntrain_file = train_df[['question_text', 'target']]\ndef text_processing(data_file):\n     stemmer = PorterStemmer()\n     #Thực hiện processing:\n     data_file['txt_processed'] = data_file['question_text'].apply(lambda train_file: word_tokenize(train_file))\n     print('step1 done...')\n     data_file['txt_processed'] = data_file['txt_processed'].apply(lambda x: [item for item in x if item.isalpha()])\n     print('step2 done..')\n     #data_file['txt_processed'] = data_file['txt_processed'].apply(lambda x: [item for item in x if item not in stop_words])\n     #print('step3 done..')\n     data_file['txt_processed'] = data_file['txt_processed'].apply(lambda x: [stemmer.stem(item) for item in x])\n     print('done')\n     return data_file\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.102655Z","iopub.execute_input":"2022-01-08T13:35:50.103846Z","iopub.status.idle":"2022-01-08T13:35:50.118399Z","shell.execute_reply.started":"2022-01-08T13:35:50.103801Z","shell.execute_reply":"2022-01-08T13:35:50.117371Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#train_file_sw = text_processing(train_file)\n#train_file_sw.tail()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.12035Z","iopub.execute_input":"2022-01-08T13:35:50.121171Z","iopub.status.idle":"2022-01-08T13:35:50.132304Z","shell.execute_reply.started":"2022-01-08T13:35:50.121126Z","shell.execute_reply":"2022-01-08T13:35:50.130956Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Xác định fscore của model**","metadata":{}},{"cell_type":"code","source":"'''\n# Hàm tính fscore của model\ndef get_fscore_matrix(fitted_clf, model_name):\n    print(model_name, ' :')\n    \n    # get classes predictions for the classification report \n    y_train_pred, y_pred = fitted_clf.predict(X_train), fitted_clf.predict(X_test)\n    print(classification_report(y_test, y_pred), '\\n') # target_names=y\n    \n    # computes probabilities keep the ones for the positive outcome only      \n    print(f'F1-score = {f1_score(y_test, y_pred):.2f}')\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.134709Z","iopub.execute_input":"2022-01-08T13:35:50.135221Z","iopub.status.idle":"2022-01-08T13:35:50.148268Z","shell.execute_reply.started":"2022-01-08T13:35:50.135177Z","shell.execute_reply":"2022-01-08T13:35:50.146566Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\n# Convert a collection of text documents to string\ntrain_file_sw['str_processed'] = train_file_sw['txt_processed'].apply(lambda x: \" \".join(x))\ntrain_file_sw.head()\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.149978Z","iopub.execute_input":"2022-01-08T13:35:50.15094Z","iopub.status.idle":"2022-01-08T13:35:50.162355Z","shell.execute_reply.started":"2022-01-08T13:35:50.150894Z","shell.execute_reply":"2022-01-08T13:35:50.160782Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\npl_model = pipeline.fit(X_train, y_train)\npl_model\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.164252Z","iopub.execute_input":"2022-01-08T13:35:50.165306Z","iopub.status.idle":"2022-01-08T13:35:50.176171Z","shell.execute_reply.started":"2022-01-08T13:35:50.165259Z","shell.execute_reply":"2022-01-08T13:35:50.174676Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#get_fscore_matrix(pl_model, 'Sử dụng Pipeline:')","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.178084Z","iopub.execute_input":"2022-01-08T13:35:50.179032Z","iopub.status.idle":"2022-01-08T13:35:50.186474Z","shell.execute_reply.started":"2022-01-08T13:35:50.178962Z","shell.execute_reply":"2022-01-08T13:35:50.184644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#pd.read_csv(\"../input/quora-insincere-questions-classification/sample_submission.csv\").head()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.188451Z","iopub.execute_input":"2022-01-08T13:35:50.18966Z","iopub.status.idle":"2022-01-08T13:35:50.196789Z","shell.execute_reply.started":"2022-01-08T13:35:50.189606Z","shell.execute_reply":"2022-01-08T13:35:50.195298Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ntest_df = text_processing(test_df)\ntest_df['str_processed'] = test_df['txt_processed'].apply(lambda x: \" \".join(x))\ntest_df.head()\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.198868Z","iopub.execute_input":"2022-01-08T13:35:50.199976Z","iopub.status.idle":"2022-01-08T13:35:50.210697Z","shell.execute_reply.started":"2022-01-08T13:35:50.199928Z","shell.execute_reply":"2022-01-08T13:35:50.209082Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"'''\ny_pred_final = pl_model.predict(test_df['str_processed'])\ndf_submission = pd.DataFrame({\"qid\":test_df[\"qid\"], \"prediction\":y_pred_final})\ndf_submission.head()\n'''","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.212696Z","iopub.execute_input":"2022-01-08T13:35:50.213744Z","iopub.status.idle":"2022-01-08T13:35:50.225662Z","shell.execute_reply.started":"2022-01-08T13:35:50.213697Z","shell.execute_reply":"2022-01-08T13:35:50.223805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#df_submission.to_csv('submission.csv', index=False)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.227607Z","iopub.execute_input":"2022-01-08T13:35:50.22865Z","iopub.status.idle":"2022-01-08T13:35:50.234154Z","shell.execute_reply.started":"2022-01-08T13:35:50.228604Z","shell.execute_reply":"2022-01-08T13:35:50.233069Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Sử dụng one-hot / LSTM**","metadata":{}},{"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom gensim.models import KeyedVectors\nimport re ## Regular expresssions\nfrom nltk import word_tokenize\nfrom sklearn import metrics\nfrom gensim.models import KeyedVectors\nimport operator\nimport gc","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.236198Z","iopub.execute_input":"2022-01-08T13:35:50.236733Z","iopub.status.idle":"2022-01-08T13:35:50.392406Z","shell.execute_reply.started":"2022-01-08T13:35:50.236688Z","shell.execute_reply":"2022-01-08T13:35:50.391153Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from tqdm import tqdm\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D,GRU\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model,load_model\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.utils.vis_utils import plot_model\nfrom keras.models import Sequential\nfrom keras.callbacks import ModelCheckpoint,EarlyStopping\nfrom tensorflow.keras.optimizers import Adam\nimport tensorflow as tf","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:50.395991Z","iopub.execute_input":"2022-01-08T13:35:50.396384Z","iopub.status.idle":"2022-01-08T13:35:51.37302Z","shell.execute_reply.started":"2022-01-08T13:35:50.396336Z","shell.execute_reply":"2022-01-08T13:35:51.372015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Chia tập dữ liệu Train thành train data và validation data**","metadata":{}},{"cell_type":"code","source":"train_data,val_data = train_test_split(train_df,test_size=0.2,stratify=train_df.target,random_state=123)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:51.374722Z","iopub.execute_input":"2022-01-08T13:35:51.375082Z","iopub.status.idle":"2022-01-08T13:35:52.963013Z","shell.execute_reply.started":"2022-01-08T13:35:51.374985Z","shell.execute_reply":"2022-01-08T13:35:52.962025Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"!unzip ../input/quora-insincere-questions-classification/embeddings.zip","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:35:52.968642Z","iopub.execute_input":"2022-01-08T13:35:52.968927Z","iopub.status.idle":"2022-01-08T13:39:26.073013Z","shell.execute_reply.started":"2022-01-08T13:35:52.968897Z","shell.execute_reply":"2022-01-08T13:39:26.071767Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def Preprocess(doc):\n    corpus=[]\n    for text in tqdm(doc):\n        text=\" \".join([contraction_fix(w) for w in text.split()])\n        text=re.sub(r'[^a-z0-9A-Z]',\" \",text) \n        text=re.sub(r'[0-9]{1}',\"#\",text) \n        text=re.sub(r'[0-9]{2}','##',text)   \n        text=re.sub(r'[0-9]{3}','###',text)\n        text=re.sub(r'[0-9]{4}','####',text)\n        text=re.sub(r'[0-9]{5,}','#####',text)\n        corpus.append(text)\n    return corpus","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:39:26.077217Z","iopub.execute_input":"2022-01-08T13:39:26.078147Z","iopub.status.idle":"2022-01-08T13:39:26.087672Z","shell.execute_reply.started":"2022-01-08T13:39:26.078077Z","shell.execute_reply":"2022-01-08T13:39:26.086307Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Xây dựng bộ từ vựng và encoding**","metadata":{}},{"cell_type":"code","source":"def vocab_build(corpus):\n    vocab={}\n    for text in tqdm(corpus):\n        for word in text.split():\n            try:\n                vocab[word]+=1\n            except KeyError:\n                vocab[word]=1\n    return vocab","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:39:26.090215Z","iopub.execute_input":"2022-01-08T13:39:26.090858Z","iopub.status.idle":"2022-01-08T13:39:30.424323Z","shell.execute_reply.started":"2022-01-08T13:39:26.090814Z","shell.execute_reply":"2022-01-08T13:39:30.423161Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def get_word_index(vocab):\n    word_index=dict((w,i+1) for i,w in enumerate(vocab.keys()))\n    return word_index\n\ndef fit_one_hot(word_index,corpus):\n    all_questions=[]\n    for text in tqdm(corpus):\n        question=[]\n        for word in text.split():\n            try:\n                question.append(word_index[word])\n            except KeyError:\n                question.append(0)\n        all_questions.append(question)\n    return all_questions","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:39:30.425938Z","iopub.execute_input":"2022-01-08T13:39:30.426766Z","iopub.status.idle":"2022-01-08T13:39:32.476115Z","shell.execute_reply.started":"2022-01-08T13:39:30.426717Z","shell.execute_reply":"2022-01-08T13:39:32.474876Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\nfile_name=\"./GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\"\nmodel_embed=KeyedVectors.load_word2vec_format(file_name,binary=True)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:39:32.477619Z","iopub.execute_input":"2022-01-08T13:39:32.47917Z","iopub.status.idle":"2022-01-08T13:40:56.754934Z","shell.execute_reply.started":"2022-01-08T13:39:32.479123Z","shell.execute_reply":"2022-01-08T13:40:56.753805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"total_text=pd.concat([train_df.question_text,test_df.question_text])\npre_text=Preprocess(total_text)\nvocabulary=vocab_build(pre_text)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:40:56.758363Z","iopub.execute_input":"2022-01-08T13:40:56.758939Z","iopub.status.idle":"2022-01-08T13:42:11.107959Z","shell.execute_reply.started":"2022-01-08T13:40:56.758891Z","shell.execute_reply":"2022-01-08T13:42:11.10692Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab_size=len(vocabulary)+1\n\nmax_len=40 #vector có độ dài cố định là 40\n\nword_index=get_word_index(vocabulary)\n\ntrain_text=Preprocess(train_data.question_text)\nval_text=Preprocess(val_data.question_text)\ntest_text=Preprocess(test_df.question_text)\n\n#encodings and paddings\nencodes=fit_one_hot(word_index,train_text) \ntrain_padded=pad_sequences(encodes,maxlen=max_len,padding=\"post\") \n\nencodes_=fit_one_hot(word_index,val_text)  \nval_padded=pad_sequences(encodes_,maxlen=max_len,padding=\"post\") \n\nencodes__=fit_one_hot(word_index,test_text) \ntest_padded=pad_sequences(encodes__,maxlen=max_len,padding=\"post\") ","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:57:31.508024Z","iopub.execute_input":"2022-01-08T13:57:31.508332Z","iopub.status.idle":"2022-01-08T13:59:08.107521Z","shell.execute_reply.started":"2022-01-08T13:57:31.5083Z","shell.execute_reply":"2022-01-08T13:59:08.106399Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"embedding_mat=np.zeros((vocab_size,300))\nfor word,i in tqdm(word_index.items()):\n    try:\n        vec=model_embed[word]\n        embedding_mat[i]=vec\n    except KeyError:\n        continue","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:59:27.206782Z","iopub.execute_input":"2022-01-08T13:59:27.207083Z","iopub.status.idle":"2022-01-08T13:59:29.007764Z","shell.execute_reply.started":"2022-01-08T13:59:27.207053Z","shell.execute_reply":"2022-01-08T13:59:29.006439Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#Tạo mô hình\ninp = Input(shape=(max_len,))\nx = Embedding(vocab_size,300,weights=[embedding_mat],input_length=max_len,trainable=False)(inp)\nx = Bidirectional(LSTM(128, return_sequences=True))(x)\nx = Conv1D(64,3,activation=\"relu\")(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(128, activation=\"relu\")(x)\nx = Dropout(0.2)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nbiLSTM = Model(inputs=inp, outputs=x)\nplot_model(biLSTM, to_file='model_plot.png', show_shapes=True, show_layer_names=True)\nprint(biLSTM.summary())","metadata":{"execution":{"iopub.status.busy":"2022-01-08T13:59:42.258141Z","iopub.execute_input":"2022-01-08T13:59:42.258926Z","iopub.status.idle":"2022-01-08T13:59:55.139914Z","shell.execute_reply.started":"2022-01-08T13:59:42.258888Z","shell.execute_reply":"2022-01-08T13:59:55.138745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"opt=Adam(learning_rate=0.001)\nbin_loss=tf.keras.losses.BinaryCrossentropy(\n                                            from_logits=False, \n                                            label_smoothing=0,\n                                            name='binary_crossentropy'\n                                        )\n\nearly_stopping=tf.keras.callbacks.EarlyStopping(\n                                                monitor=\"val_loss\",\n                                                patience=3,\n                                                mode=\"min\",\n                                                restore_best_weights=True\n                                              )\n\nreduce_lr=tf.keras.callbacks.ReduceLROnPlateau(\n                                                monitor=\"val_loss\",\n                                                factor=0.2,\n                                                patience=2,\n                                                verbose=1,\n                                                mode=\"auto\"\n                                            )\n\nmy_callbacks=[early_stopping,reduce_lr]","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:00:00.431605Z","iopub.execute_input":"2022-01-08T14:00:00.432039Z","iopub.status.idle":"2022-01-08T14:00:00.44498Z","shell.execute_reply.started":"2022-01-08T14:00:00.431961Z","shell.execute_reply":"2022-01-08T14:00:00.442402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"biLSTM.compile(loss=bin_loss, optimizer=opt, metrics=['accuracy'])\ntrained = biLSTM.fit(train_padded, train_data.target, batch_size=512, epochs=30, validation_data=(val_padded, val_data.target),callbacks=my_callbacks)","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:00:06.812653Z","iopub.execute_input":"2022-01-08T14:00:06.812941Z","iopub.status.idle":"2022-01-08T14:07:01.69378Z","shell.execute_reply.started":"2022-01-08T14:00:06.812911Z","shell.execute_reply":"2022-01-08T14:07:01.692558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\n#accuracy của mô hình\nplt.plot(trained.history['accuracy'])\nplt.plot(trained.history['val_accuracy'])\nplt.title('model accuracy')\nplt.ylabel('accuracy')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()\n\n#loss của mô hình\nplt.plot(trained.history['loss'])\nplt.plot(trained.history['val_loss'])\nplt.title('model loss')\nplt.ylabel('loss')\nplt.xlabel('epoch')\nplt.legend(['train', 'test'], loc='upper left')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:08:15.281465Z","iopub.execute_input":"2022-01-08T14:08:15.281818Z","iopub.status.idle":"2022-01-08T14:08:32.87385Z","shell.execute_reply.started":"2022-01-08T14:08:15.28177Z","shell.execute_reply":"2022-01-08T14:08:32.872883Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"y_pre = biLSTM.predict(val_padded)\nscores_f1 = []\nthreshold = []\n\nfor thresh in np.arange(0.1,0.5,0.01):\n    f1= metrics.f1_score(val_data.target,(y_pre>thresh).astype(int))\n    scores_f1.append(f1)\n    threshold.append(thresh)\n    print(\"threshold {0:2.2f} f1 score:{1:2.3f}\".format(thresh,f1))\n    \nplt.plot(threshold,scores_f1)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:09:04.636266Z","iopub.execute_input":"2022-01-08T14:09:04.636576Z","iopub.status.idle":"2022-01-08T14:09:40.420103Z","shell.execute_reply.started":"2022-01-08T14:09:04.636535Z","shell.execute_reply":"2022-01-08T14:09:40.418979Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"threshold = 0.375\ny_test_pre = biLSTM.predict(test_padded)\ny_test_pre = (y_test_pre>thresh).astype(int)\n\nsubmit=pd.DataFrame()\nsubmit[\"qid\"]=test_df.qid\nsubmit[\"prediction\"]=y_test_pre\nsubmit.to_csv(\"submission.csv\",index=False)\n","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:14:52.195143Z","iopub.execute_input":"2022-01-08T14:14:52.195441Z","iopub.status.idle":"2022-01-08T14:16:15.101018Z","shell.execute_reply.started":"2022-01-08T14:14:52.19541Z","shell.execute_reply":"2022-01-08T14:16:15.100061Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#print(y_test_pre[2])","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:17:22.829152Z","iopub.execute_input":"2022-01-08T14:17:22.829437Z","iopub.status.idle":"2022-01-08T14:17:22.837817Z","shell.execute_reply.started":"2022-01-08T14:17:22.82941Z","shell.execute_reply":"2022-01-08T14:17:22.836602Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Sử dụng pre-trained model BERT**","metadata":{}},{"cell_type":"code","source":"#from transformers import AutoTokenizer, AutoModelForSequenceClassification","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:47:46.523245Z","iopub.execute_input":"2022-01-08T14:47:46.523522Z","iopub.status.idle":"2022-01-08T14:47:46.529140Z","shell.execute_reply.started":"2022-01-08T14:47:46.523493Z","shell.execute_reply":"2022-01-08T14:47:46.527739Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#tokenizer = AutoTokenizer.from_pretrained('bert-base-uncased')","metadata":{"execution":{"iopub.status.busy":"2022-01-08T14:55:25.602190Z","iopub.execute_input":"2022-01-08T14:55:25.602733Z","iopub.status.idle":"2022-01-08T14:55:45.686698Z","shell.execute_reply.started":"2022-01-08T14:55:25.602696Z","shell.execute_reply":"2022-01-08T14:55:45.683880Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"class QuoraDataset(Dataset):\n    def __init__(self, X, y, tokenizer):\n        self.text = X.reset_index(drop=True)\n        self.targets = y.reset_index(drop=True)\n        self.tok = tokenizer\n    def __len__(self):\n        return len(self.text)\n    \n    def __getitem__(self, idx):\n        \n        text = self.text[idx]\n        targ = self.targets[idx]\n        \n        return self.tok(text, padding='max_length', \n                        truncation=True,\n                        max_length=30,\n                        return_tensors=\"pt\")[\"input_ids\"][0], tensor(targ)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Split data\nX_train, X_valid, y_train, y_valid = train_test_split(train_df[\"question_text\"], train_df[\"target\"], \n                                                      stratify=train_df[\"target\"],  test_size=0.1)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_ds = QuoraDataset(X_train, y_train, tokenizer) \nvalid_ds = QuoraDataset(X_valid, y_valid, tokenizer)","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Prepare data loader\ntrain_dl = DataLoader(train_ds, bs=128)\nvalid_dl = DataLoader(valid_ds, bs=256)\ndata_loader = DataLoaders(train_dl, valid_dl).to(\"cuda\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Xây dựng BERT model\n\nbert = AutoModelForSequenceClassification.from_pretrained('bert-base-cased').train()\nclassifier = nn.Sequential(\n    nn.Linear(768, 1024),\n    nn.ReLU(),\n    nn.Dropout(0.5),\n    nn.Linear(1024, 2)\n)\nbert.classifier = classifier\nclass BertClassifier(Module):\n    def __init__(self, bert):\n        self.bert = bert\n    def forward(self, x):\n        x = self.bert(x)\n        return x.logits\n\nmodel = BertClassifier(bert).to(\"cuda\")","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"num_sin = (train_file[\"target\"] == 0).sum() # số câu hỏi chân thành\nnum_insin = (train_file[\"target\"] == 1).sum() # Số câu hỏi không chân thành\nn = num_sin + num_insin\nclass_weights = tensor([n / (n+num_sin), n / (n+num_insin)]).to('cuda')\nlearn = Learner(data_loader, model, \n                loss_func=nn.CrossEntropyLoss(weight=class_weights), \n                metrics=[accuracy, F1Score()]).to_fp16()\nlearn.lr_find() # Biểu thị bảng learning rate","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"learn.fit_one_cycle(2, lr_max=2e-5)","metadata":{},"execution_count":null,"outputs":[]}]}