{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Tổng quan vấn đề\n\n* Một vấn đề tồn tại với bất kì trang web lớn nào hiện nay là làm thế nào xử lý nội dung độc hại. Quora muốn giải quyết vấn đề này trực tiếp để giữ cho nền tảng của họ trở thành một nơi mà người dùng có thể cảm thấy an toàn khi chia sẻ kiến thức của họ với thế giới.\n* Quora là một nền tảng cho phép mọi người học hỏi lẫn nhau. Trên Quora, mọi người có thể đặt câu hỏi và kết nối với những người khác, những người đóng góp thông tin chi tiết độc đáo và câu trả lời chất lượng. Một thách thức quan trọng là loại bỏ những câu hỏi insincere - những câu hỏi được đặt ra dựa trên những tiền đề sai lầm hoặc có ý định đưa ra một tuyên bố hơn là tìm kiếm câu trả lời hữu ích.\n* Mục tiêu: loại bỏ những câu hỏi insincere\n","metadata":{}},{"cell_type":"markdown","source":"# Thực hiện bài toán","metadata":{}},{"cell_type":"code","source":"import numpy as np \nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport re\nimport os\n\nfrom tqdm import tqdm\nfrom collections import Counter\nfrom wordcloud import WordCloud, STOPWORDS\nfrom scipy.sparse import hstack\nfrom IPython.display import Image\nfrom tqdm import tqdm_notebook\n\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import PorterStemmer, SnowballStemmer, WordNetLemmatizer\nfrom nltk.stem.lancaster import LancasterStemmer\nfrom nltk.util import ngrams\nprint(os.listdir(\"../input\"))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:33.520949Z","iopub.execute_input":"2021-05-25T14:54:33.521473Z","iopub.status.idle":"2021-05-25T14:54:34.226558Z","shell.execute_reply.started":"2021-05-25T14:54:33.521357Z","shell.execute_reply":"2021-05-25T14:54:34.225633Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')\nsub = pd.read_csv('../input/quora-insincere-questions-classification/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:34.228204Z","iopub.execute_input":"2021-05-25T14:54:34.228864Z","iopub.status.idle":"2021-05-25T14:54:39.424279Z","shell.execute_reply.started":"2021-05-25T14:54:34.228814Z","shell.execute_reply":"2021-05-25T14:54:39.423449Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.info()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.425885Z","iopub.execute_input":"2021-05-25T14:54:39.426362Z","iopub.status.idle":"2021-05-25T14:54:39.690220Z","shell.execute_reply.started":"2021-05-25T14:54:39.426327Z","shell.execute_reply":"2021-05-25T14:54:39.688985Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"- Dữ liệu train bao gồm 13 triệu dòng và 3 cột\n- Các trường dữ liệu:\n+ qid: mã định danh\n+ question_text: nội dung câu hỏi\n+ target: câu hỏi sincere khi có giá trị 0 và insincere khi có giá trị 1","metadata":{}},{"cell_type":"code","source":"train.head(10)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.692750Z","iopub.execute_input":"2021-05-25T14:54:39.693229Z","iopub.status.idle":"2021-05-25T14:54:39.713128Z","shell.execute_reply.started":"2021-05-25T14:54:39.693181Z","shell.execute_reply":"2021-05-25T14:54:39.711813Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[\"target\"].value_counts()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.715065Z","iopub.execute_input":"2021-05-25T14:54:39.715588Z","iopub.status.idle":"2021-05-25T14:54:39.737275Z","shell.execute_reply.started":"2021-05-25T14:54:39.715541Z","shell.execute_reply":"2021-05-25T14:54:39.736095Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Dữ liệu train gồm 80810 dòng đã được xác nhận là câu hỏi insincere và 1225312 dòng là sincere","metadata":{}},{"cell_type":"code","source":"fig = plt.figure()\nax = fig.add_axes([0,0,2,1])\ntype_check = ['sincere','insincere']\ncount = [1225312,80810]\nax.bar(type_check,count)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.739891Z","iopub.execute_input":"2021-05-25T14:54:39.740211Z","iopub.status.idle":"2021-05-25T14:54:39.872176Z","shell.execute_reply.started":"2021-05-25T14:54:39.740182Z","shell.execute_reply":"2021-05-25T14:54:39.870977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('- Phần trăm câu hỏi sincere (target = 0): {}%'.format(100 - round(train['target'].mean() * 100, 2))) \nprint('- Phần trăm câu hỏi insincere (target = 1): {}%'.format(round(train['target'].mean() * 100, 2)))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.873576Z","iopub.execute_input":"2021-05-25T14:54:39.873927Z","iopub.status.idle":"2021-05-25T14:54:39.884302Z","shell.execute_reply.started":"2021-05-25T14:54:39.873896Z","shell.execute_reply":"2021-05-25T14:54:39.882874Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"****","metadata":{}},{"cell_type":"markdown","source":"\n# Phân tích dữ liệu ","metadata":{}},{"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[',\n          ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', '·', '_', '{', '}',\n          '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′',\n          'Â', '█', '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬',\n          '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒',\n          '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦',\n          '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»',\n          '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤',\n          '‡', '√' ]\n","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.886880Z","iopub.execute_input":"2021-05-25T14:54:39.887169Z","iopub.status.idle":"2021-05-25T14:54:39.897490Z","shell.execute_reply.started":"2021-05-25T14:54:39.887141Z","shell.execute_reply":"2021-05-25T14:54:39.896164Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loại bỏ các kí tự đặc biệt\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.900821Z","iopub.execute_input":"2021-05-25T14:54:39.901196Z","iopub.status.idle":"2021-05-25T14:54:39.910343Z","shell.execute_reply.started":"2021-05-25T14:54:39.901164Z","shell.execute_reply":"2021-05-25T14:54:39.909196Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loại bỏ số\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.911759Z","iopub.execute_input":"2021-05-25T14:54:39.912145Z","iopub.status.idle":"2021-05-25T14:54:39.923152Z","shell.execute_reply.started":"2021-05-25T14:54:39.912107Z","shell.execute_reply":"2021-05-25T14:54:39.921882Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"mispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"that's\" : \"that is\",\n\"there's\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.924652Z","iopub.execute_input":"2021-05-25T14:54:39.924945Z","iopub.status.idle":"2021-05-25T14:54:39.937807Z","shell.execute_reply.started":"2021-05-25T14:54:39.924918Z","shell.execute_reply":"2021-05-25T14:54:39.936712Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Thay thế các từ viết tắt\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.939328Z","iopub.execute_input":"2021-05-25T14:54:39.939645Z","iopub.status.idle":"2021-05-25T14:54:39.959162Z","shell.execute_reply.started":"2021-05-25T14:54:39.939592Z","shell.execute_reply":"2021-05-25T14:54:39.957855Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Loại bỏ các từ trong stopword\nstopword_list = nltk.corpus.stopwords.words('english')\ndef remove_stopwords(text, is_lower_case=True):\n    tokenizer = ToktokTokenizer()\n    tokens = tokenizer.tokenize(text)\n    tokens = [token.strip() for token in tokens]\n    if is_lower_case:\n        filtered_tokens = [token for token in tokens if token not in stopword_list]\n    else:\n        filtered_tokens = [token for token in tokens if token.lower() not in stopword_list]\n    filtered_text = ' '.join(filtered_tokens)\n    return filtered_text","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.960723Z","iopub.execute_input":"2021-05-25T14:54:39.961138Z","iopub.status.idle":"2021-05-25T14:54:39.975075Z","shell.execute_reply.started":"2021-05-25T14:54:39.961095Z","shell.execute_reply":"2021-05-25T14:54:39.974242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Stemming\nfrom nltk.stem import SnowballStemmer\nfrom nltk.tokenize.toktok import ToktokTokenizer\ndef stem_text(text):\n    tokenizer = ToktokTokenizer()\n    stemmer = SnowballStemmer('english')\n    tokens = tokenizer.tokenize(text)\n    tokens = [token.strip() for token in tokens]\n    tokens = [stemmer.stem(token) for token in tokens]\n    return ' '.join(tokens)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.976498Z","iopub.execute_input":"2021-05-25T14:54:39.977133Z","iopub.status.idle":"2021-05-25T14:54:39.993779Z","shell.execute_reply.started":"2021-05-25T14:54:39.977086Z","shell.execute_reply":"2021-05-25T14:54:39.992801Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Lemmatization\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize.toktok import ToktokTokenizer\nwordnet_lemmatizer = WordNetLemmatizer()\ndef lemma_text(text):\n    tokenizer = ToktokTokenizer()\n    tokens = tokenizer.tokenize(text)\n    tokens = [token.strip() for token in tokens]\n    tokens = [wordnet_lemmatizer.lemmatize(token) for token in tokens]\n    return ' '.join(tokens)","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:39.995164Z","iopub.execute_input":"2021-05-25T14:54:39.995819Z","iopub.status.idle":"2021-05-25T14:54:40.007741Z","shell.execute_reply.started":"2021-05-25T14:54:39.995770Z","shell.execute_reply":"2021-05-25T14:54:40.006783Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def clean_sentence(x):\n    x = x.lower()\n    x = clean_text(x)\n    x = clean_numbers(x)\n    x = replace_typical_misspell(x)\n    x = remove_stopwords(x)\n    x = stem_text(x)\n    x = lemma_text(x)\n    x = x.replace(\"'\",\"\")\n    return x","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:40.008952Z","iopub.execute_input":"2021-05-25T14:54:40.009369Z","iopub.status.idle":"2021-05-25T14:54:40.021419Z","shell.execute_reply.started":"2021-05-25T14:54:40.009338Z","shell.execute_reply":"2021-05-25T14:54:40.020162Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Xử lí dữ liệu trên cả tập train và tập test \ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_sentence(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_sentence(x))","metadata":{"execution":{"iopub.status.busy":"2021-05-25T14:54:40.023306Z","iopub.execute_input":"2021-05-25T14:54:40.023828Z","iopub.status.idle":"2021-05-25T15:07:49.008260Z","shell.execute_reply.started":"2021-05-25T14:54:40.023779Z","shell.execute_reply":"2021-05-25T15:07:49.006155Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# word cloud cho câu hỏi sincere\nsincere_wordcloud = WordCloud().generate(str(train[train[\"target\"] == 0][\"question_text\"]))\nplt.figure(figsize=(8,7))\nplt.imshow(sincere_wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T15:07:49.010944Z","iopub.execute_input":"2021-05-25T15:07:49.011435Z","iopub.status.idle":"2021-05-25T15:07:49.506356Z","shell.execute_reply.started":"2021-05-25T15:07:49.011384Z","shell.execute_reply":"2021-05-25T15:07:49.505353Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# word cloud cho câu hỏi insincere\nsincere_wordcloud = WordCloud().generate(str(train[train[\"target\"] == 1][\"question_text\"]))\nplt.figure(figsize=(8,7))\nplt.imshow(sincere_wordcloud)\nplt.axis(\"off\")\nplt.tight_layout(pad=0)\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-05-25T15:07:49.507973Z","iopub.execute_input":"2021-05-25T15:07:49.508523Z","iopub.status.idle":"2021-05-25T15:07:49.861063Z","shell.execute_reply.started":"2021-05-25T15:07:49.508479Z","shell.execute_reply":"2021-05-25T15:07:49.859968Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.model_selection import KFold\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import MultinomialNB\nfrom sklearn.metrics import accuracy_score, f1_score, confusion_matrix, classification_report\n\nX_train, X_test, y_train, y_test = train_test_split(train['question_text'], train['target'],\ntest_size=0.3)\nprint('x_train: ', X_train.shape, y_train.shape)\nprint('x_test: ',X_test.shape, y_test.shape)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-25T15:07:49.862717Z","iopub.execute_input":"2021-05-25T15:07:49.863309Z","iopub.status.idle":"2021-05-25T15:07:50.554660Z","shell.execute_reply.started":"2021-05-25T15:07:49.863242Z","shell.execute_reply":"2021-05-25T15:07:50.553480Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vectorizer = CountVectorizer()\nvectorizer.fit(list(X_train) + list(X_test))\nX_train = vectorizer.transform(X_train) \nX_test = vectorizer.transform(X_test)\n","metadata":{"execution":{"iopub.status.busy":"2021-05-25T15:07:50.556256Z","iopub.execute_input":"2021-05-25T15:07:50.556894Z","iopub.status.idle":"2021-05-25T15:08:28.691082Z","shell.execute_reply.started":"2021-05-25T15:07:50.556845Z","shell.execute_reply":"2021-05-25T15:08:28.686787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"logistic = LogisticRegression() \nlogistic.fit(X_train,y_train)\ntrain_predictions = logistic.predict(X_train)\ntrain_acc = accuracy_score(y_train, train_predictions)  \ntrain_f1 = f1_score(y_train, train_predictions) \nprint(f\"Training accuracy: {train_acc:.2%}, F1: {train_f1:.4f}\") \ntest_predictions = logistic.predict(X_test)\ntest_acc = accuracy_score(y_test, test_predictions) \ntest_f1 = f1_score(y_test, test_predictions) \nprint(f\"Testing accuracy:  {test_acc:.2%}, F1: {test_f1:.4f}\")","metadata":{"execution":{"iopub.status.busy":"2021-05-25T15:08:28.693374Z","iopub.execute_input":"2021-05-25T15:08:28.693895Z","iopub.status.idle":"2021-05-25T15:08:55.535389Z","shell.execute_reply.started":"2021-05-25T15:08:28.693846Z","shell.execute_reply":"2021-05-25T15:08:55.534303Z"},"trusted":true},"execution_count":null,"outputs":[]}]}