{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# **Introduction**\nClassifying Quora questions whether they are insincere or sincere ones","metadata":{}},{"cell_type":"markdown","source":"Input: A csv file with texts
\nOutput: (0, 1) = (sincere, insincere)","metadata":{}},{"cell_type":"markdown","source":"# **Import necessary libraries**\n# **Import spacy**","metadata":{}},{"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n\n%matplotlib inline\nimport warnings\nwarnings.filterwarnings('ignore')\n\n#import necessary libraries\nimport os\nimport csv\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport string\n\n#import spacy\nimport re\nimport nltk\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom PIL import Image\nfrom wordcloud import WordCloud #tag cloud: novelty visual representation of text data","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:39.245335Z","iopub.execute_input":"2021-06-10T17:54:39.245741Z","iopub.status.idle":"2021-06-10T17:54:40.697209Z","shell.execute_reply.started":"2021-06-10T17:54:39.245711Z","shell.execute_reply":"2021-06-10T17:54:40.696271Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Starting to understand the input files**","metadata":{}},{"cell_type":"markdown","source":"# **Load data into dataframe then print out to observe**","metadata":{}},{"cell_type":"markdown","source":"Read input files
\nUsing pandas: CSV file input/output","metadata":{}},{"cell_type":"code","source":"train_data = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest_data = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:40.698805Z","iopub.execute_input":"2021-06-10T17:54:40.699155Z","iopub.status.idle":"2021-06-10T17:54:46.261494Z","shell.execute_reply.started":"2021-06-10T17:54:40.699120Z","shell.execute_reply":"2021-06-10T17:54:46.260645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out some of the first data in train.csv file**\n+ The raw data contains 1306122 rows and 3 columns
\n+ The feature includes \"questions id\", \"questions text\", \"target\"
\n+ Questions id: Id of the question, qid may not take part in classfying questions -> can ignore
\n+ Questions text: Since this field is the only one that directly affects the subclass of the question, preprocessing is required
\n+ Target: Sincere question target = 0; Insincere question target = 1
\n+ Can not see the questions classification yet","metadata":{}},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:46.264357Z","iopub.execute_input":"2021-06-10T17:54:46.264998Z","iopub.status.idle":"2021-06-10T17:54:46.285104Z","shell.execute_reply.started":"2021-06-10T17:54:46.264946Z","shell.execute_reply":"2021-06-10T17:54:46.284007Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out some of the first data in test.csv file**\n+ The raw data contains 375806 rows and 2 columns
\n+ The feature includes questions id, questions text
\n--> These questions are the ones we have to set target (0, 1), which is the purpose of this challenge","metadata":{}},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:46.286957Z","iopub.execute_input":"2021-06-10T17:54:46.287340Z","iopub.status.idle":"2021-06-10T17:54:46.300802Z","shell.execute_reply.started":"2021-06-10T17:54:46.287304Z","shell.execute_reply":"2021-06-10T17:54:46.299854Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(\"Dimensions of Training Dataset : \", train_data.shape)\nprint(\"Dimensions of Test Dataset : \", test_data.shape)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:46.302212Z","iopub.execute_input":"2021-06-10T17:54:46.302627Z","iopub.status.idle":"2021-06-10T17:54:46.312221Z","shell.execute_reply.started":"2021-06-10T17:54:46.302590Z","shell.execute_reply":"2021-06-10T17:54:46.311319Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sns.countplot(x=\"target\", data=train_data, palette=\"Set1\")\nplt.title('Target Count')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:46.313573Z","iopub.execute_input":"2021-06-10T17:54:46.314185Z","iopub.status.idle":"2021-06-10T17:54:46.499612Z","shell.execute_reply.started":"2021-06-10T17:54:46.314141Z","shell.execute_reply":"2021-06-10T17:54:46.498651Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"--> The number of sincere questions are much greater than the number of insincere questions","metadata":{}},{"cell_type":"markdown","source":"# **Print out the number of sincere questions and insincere questions**\n+ Sincere questions have the target tag = 0
\n+ Insincere questions have the target tag = 1","metadata":{}},{"cell_type":"code","source":"num_questions = len(train_data['qid'])\nnum_sincere_questions = len(train_data.qid[train_data['target'] == 0])\nnum_insincere_questions = len(train_data.qid[train_data['target'] == 1])\nprint(\"Number of Sincere questions in the training set : \", num_sincere_questions)\nprint(\"Number of Insincere questions in the training set : \", num_insincere_questions)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:46.501018Z","iopub.execute_input":"2021-06-10T17:54:46.501358Z","iopub.status.idle":"2021-06-10T17:54:46.573298Z","shell.execute_reply.started":"2021-06-10T17:54:46.501320Z","shell.execute_reply":"2021-06-10T17:54:46.571455Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out the graph to see the classification**","metadata":{}},{"cell_type":"code","source":"values = [train_data[train_data['target']==0].shape[0], train_data[train_data['target']==1].shape[0]]\nlabels = ['Sincere Questions', 'Insincere Questions']\n\nplt.pie(values, labels=labels, autopct='%1.1f%%', shadow=True)\nplt.title('Target Distribution')\nplt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:46.574623Z","iopub.execute_input":"2021-06-10T17:54:46.574957Z","iopub.status.idle":"2021-06-10T17:54:46.790063Z","shell.execute_reply.started":"2021-06-10T17:54:46.574905Z","shell.execute_reply":"2021-06-10T17:54:46.789105Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(num_sincere_questions/num_questions * 100, 'percent of training data questions are sincere')\nprint(num_insincere_questions/num_questions * 100, 'percent of training data questions are insincere')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:46.793163Z","iopub.execute_input":"2021-06-10T17:54:46.793515Z","iopub.status.idle":"2021-06-10T17:54:46.799582Z","shell.execute_reply.started":"2021-06-10T17:54:46.793478Z","shell.execute_reply":"2021-06-10T17:54:46.798441Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Based on the above graph, we get the data divided into 2 classes: 0 = sincere; 1 = insincere**
\n**+ class 0 : 1225312 data accounts for 93.81%**
\n**+ class 1: 80810 data accounts for 6.19%**
\n**--> The dataset for training is unbalanced (sincere questions are 15 times more insincere ones)**","metadata":{}},{"cell_type":"markdown","source":"**Data ratio 15:1 will often lead to misinterpretation of model quality. Then the model evaluation measure is the accuracy which can be achieved very high without the model (which is 93%)**
\n**--> Accuracy should not be selected as the model evaluation index to avoid false optimism about quality**
\n**--> We will use the F1_score to evaluate the model**","metadata":{}},{"cell_type":"markdown","source":"**F1_score is the harmonic mean between precision and recall**
\n\n**Precision: in the found set, how many classified questions are correct**
\n\n**Recall: of the existence, how many can be found (category)**","metadata":{}},{"cell_type":"code","source":"train_data.isnull().sum()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:46.801832Z","iopub.execute_input":"2021-06-10T17:54:46.802234Z","iopub.status.idle":"2021-06-10T17:54:47.017588Z","shell.execute_reply.started":"2021-06-10T17:54:46.802189Z","shell.execute_reply":"2021-06-10T17:54:47.016646Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data.duplicated(subset = [\"question_text\", \"qid\", \"target\"]).any()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:47.018978Z","iopub.execute_input":"2021-06-10T17:54:47.019335Z","iopub.status.idle":"2021-06-10T17:54:47.884538Z","shell.execute_reply.started":"2021-06-10T17:54:47.019300Z","shell.execute_reply":"2021-06-10T17:54:47.883561Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out the info of the train data and test data**","metadata":{}},{"cell_type":"code","source":"train_data.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:47.886061Z","iopub.execute_input":"2021-06-10T17:54:47.886402Z","iopub.status.idle":"2021-06-10T17:54:48.109722Z","shell.execute_reply.started":"2021-06-10T17:54:47.886364Z","shell.execute_reply":"2021-06-10T17:54:48.108727Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.info()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:48.111356Z","iopub.execute_input":"2021-06-10T17:54:48.111725Z","iopub.status.idle":"2021-06-10T17:54:48.181577Z","shell.execute_reply.started":"2021-06-10T17:54:48.111684Z","shell.execute_reply":"2021-06-10T17:54:48.180779Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Use Natural Language Tootkit to clean the data**\n+ Wordnet: It groups English words into sets of synonyms called synonym series, provides brief definitions and usage examples, and records the number of relationships between these synonym series or members
\n+ Punkt: Punkt Sentence Tokenizer. This tokenizer divides a text into a list of sentences, by using an unsupervised algorithm to build a model for abbreviation words, collocations, and words that start sentences
\n+ Stopwords: For the purpose of analyzing text data and building NLP models, these stopwords might not add much value to the meaning of the document --> We have to remove these words","metadata":{}},{"cell_type":"code","source":"nltk.download('wordnet')\nnltk.download('punkt')\nnltk.download('stopwords')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:48.182996Z","iopub.execute_input":"2021-06-10T17:54:48.183435Z","iopub.status.idle":"2021-06-10T17:54:48.426911Z","shell.execute_reply.started":"2021-06-10T17:54:48.183400Z","shell.execute_reply":"2021-06-10T17:54:48.426101Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Process the raw data text to see the categories in the sentence**\n# **Print out some of the first data**","metadata":{}},{"cell_type":"code","source":"train_data['freq_qid'] = train_data.groupby('qid')['qid'].transform('count') \ntrain_data['qlen'] = train_data['question_text'].str.len() \ntrain_data['n_words'] = train_data['question_text'].apply(lambda row: len(row.split(\" \")))\ntrain_data['numeric_words'] = train_data['question_text'].apply(lambda row: sum(c.isdigit() for c in row))\ntrain_data['sp_char_words'] = train_data['question_text'].str.findall(r'[^a-zA-Z0–9 ]').str.len()\ntrain_data['char_words'] = train_data['question_text'].apply(lambda row: len(str(row)))\ntrain_data['unique_words'] = train_data['question_text'].apply(lambda row: len(set(str(row).split())))\n\ntrain_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:54:48.428317Z","iopub.execute_input":"2021-06-10T17:54:48.428882Z","iopub.status.idle":"2021-06-10T17:55:08.778615Z","shell.execute_reply.started":"2021-06-10T17:54:48.428846Z","shell.execute_reply":"2021-06-10T17:55:08.777848Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['freq_qid'] = test_data.groupby('qid')['qid'].transform('count') \ntest_data['qlen'] = test_data['question_text'].str.len() \ntest_data['n_words'] = test_data['question_text'].apply(lambda row: len(row.split(\" \")))\ntest_data['numeric_words'] = test_data['question_text'].apply(lambda row: sum(c.isdigit() for c in row))\ntest_data['sp_char_words'] = test_data['question_text'].str.findall(r'[^a-zA-Z0–9 ]').str.len()\ntest_data['char_words'] = test_data['question_text'].apply(lambda row: len(str(row)))\ntest_data['unique_words'] = test_data['question_text'].apply(lambda row: len(set(str(row).split())))\n\ntest_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:08.779966Z","iopub.execute_input":"2021-06-10T17:55:08.780322Z","iopub.status.idle":"2021-06-10T17:55:14.312372Z","shell.execute_reply.started":"2021-06-10T17:55:08.780284Z","shell.execute_reply":"2021-06-10T17:55:14.311374Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Note: Toxic questions have average of more words and number than non-toxic questions**\n# **Insincere questions usually have bad meaning words rathar than grammar**\n**Can use this feature into model (Tested but no good with linear models)**","metadata":{}},{"cell_type":"markdown","source":"---------------------------------------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"# **Special letters, numbers, paths, uppercase or lowercase usually do not affect the classification of the question, so they can be omitted**","metadata":{}},{"cell_type":"markdown","source":"****","metadata":{}},{"cell_type":"markdown","source":"**Remove special characters**","metadata":{}},{"cell_type":"code","source":"puncts=[',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', \n '•', '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', \n '█', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '¬', '░', '¡', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', \n '—', '‹', '─', '▒', '：', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', '≤', '‡', '√', '◄', '━', \n '⇒', '▶', '≥', '╝', '♡', '◊', '。', '✈', '≡', '☺', '✔', '↵', '≈', '✓', '♣', '☎', '℃', '◦', '└', '‟', '～', '！', '○', \n '◆', '№', '♠', '▌', '✿', '▸', '⁄', '□', '❖', '✦', '．', '÷', '｜', '┃', '／', '￥', '╠', '↩', '✭', '▐', '☼', '☻', '┐', \n '├', '«', '∼', '┌', '℉', '☮', '฿', '≦', '♬', '✧', '〉', '－', '⌂', '✖', '･', '◕', '※', '‖', '◀', '‰', '\\x97', '↺', \n '∆', '┘', '┬', '╬', '،', '⌘', '⊂', '＞', '〈', '⎙', '？', '☠', '⇐', '▫', '∗', '∈', '≠', '♀', '♔', '˚', '℗', '┗', '＊', \n '┼', '❀', '＆', '∩', '♂', '‿', '∑', '‣', '➜', '┛', '⇓', '☯', '⊖', '☀', '┳', '；', '∇', '⇑', '✰', '◇', '♯', '☞', '´', \n '↔', '┏', '｡', '◘', '∂', '✌', '♭', '┣', '┴', '┓', '✨', '\\xa0', '˜', '❥', '┫', '℠', '✒', '［', '∫', '\\x93', '≧', '］', \n '\\x94', '∀', '♛', '\\x96', '∨', '◎', '↻', '⇩', '＜', '≫', '✩', '✪', '♕', '؟', '₤', '☛', '╮', '␊', '＋', '┈', '％', \n '╋', '▽', '⇨', '┻', '⊗', '￡', '।', '▂', '✯', '▇', '＿', '➤', '✞', '＝', '▷', '△', '◙', '▅', '✝', '∧', '␉', '☭', \n '┊', '╯', '☾', '➔', '∴', '\\x92', '▃', '↳', '＾', '׳', '➢', '╭', '➡', '＠', '⊙', '☢', '˝', '∏', '„', '∥', '❝', '☐', \n '▆', '╱', '⋙', '๏', '☁', '⇔', '▔', '\\x91', '➚', '◡', '╰', '\\x85', '♢', '˙', '۞', '✘', '✮', '☑', '⋆', 'ⓘ', '❒', \n '☣', '✉', '⌊', '➠', '∣', '❑', '◢', 'ⓒ', '\\x80', '〒', '∕', '▮', '⦿', '✫', '✚', '⋯', '♩', '☂', '❞', '‗', '܂', '☜', \n '‾', '✜', '╲', '∘', '⟩', '＼', '⟨', '·', '✗', '♚', '∅', 'ⓔ', '◣', '͡', '‛', '❦', '◠', '✄', '❄', '∃', '␣', '≪', '｢', \n '≅', '◯', '☽', '∎', '｣', '❧', '̅', 'ⓐ', '↘', '⚓', '▣', '˘', '∪', '⇢', '✍', '⊥', '＃', '⎯', '↠', '۩', '☰', '◥', \n '⊆', '✽', '⚡', '↪', '❁', '☹', '◼', '☃', '◤', '❏', 'ⓢ', '⊱', '➝', '̣', '✡', '∠', '｀', '▴', '┤', '∝', '♏', 'ⓐ', \n '✎', ';', '', '＇', '❣', '✂', '✤', 'ⓞ', '☪', '✴', '⌒', '˛', '♒', '＄', '✶', '▻', 'ⓔ', '◌', '◈', '❚', '❂', '￦', \n '◉', '╜', '̃', '✱', '╖', '❉', 'ⓡ', '↗', 'ⓣ', '♻', '➽', '׀', '✲', '✬', '☉', '▉', '≒', '☥', '⌐', '♨', '✕', 'ⓝ', \n '⊰', '❘', '＂', '⇧', '̵', '➪', '▁', '▏', '⊃', 'ⓛ', '‚', '♰', '́', '✏', '⏑', '̶', 'ⓢ', '⩾', '￠', '❍', '≃', '⋰', '♋', \n '､', '̂', '❋', '✳', 'ⓤ', '╤', '▕', '⌣', '✸', '℮', '⁺', '▨', '╨', 'ⓥ', '♈', '❃', '☝', '✻', '⊇', '≻', '♘', '♞', \n '◂', '✟', '⌠', '✠', '☚', '✥', '❊', 'ⓒ', '⌈', '❅', 'ⓡ', '♧', 'ⓞ', '▭', '❱', 'ⓣ', '∟', '☕', '♺', '∵', '⍝', 'ⓑ', \n '✵', '✣', '٭', '♆', 'ⓘ', '∶', '⚜', '◞', '்', '✹', '➥', '↕', '̳', '∷', '✋', '➧', '∋', '̿', 'ͧ', '┅', '⥤', '⬆', '⋱', \n '☄', '↖', '⋮', '۔', '♌', 'ⓛ', '╕', '♓', '❯', '♍', '▋', '✺', '⭐', '✾', '♊', '➣', '▿', 'ⓑ', '♉', '⏠', '◾', '▹', \n '⩽', '↦', '╥', '⍵', '⌋', '։', '➨', '∮', '⇥', 'ⓗ', 'ⓓ', '⁻', '⎝', '⌥', '⌉', '◔', '◑', '✼', '♎', '♐', '╪', '⊚', \n '☒', '⇤', 'ⓜ', '⎠', '◐', '⚠', '╞', '◗', '⎕', 'ⓨ', '☟', 'ⓟ', '♟', '❈', '↬', 'ⓓ', '◻', '♮', '❙', '♤', '∉', '؛', \n '⁂', 'ⓝ', '־', '♑', '╫', '╓', '╳', '⬅', '☔', '☸', '┄', '╧', '׃', '⎢', '❆', '⋄', '⚫', '̏', '☏', '➞', '͂', '␙', \n 'ⓤ', '◟', '̊', '⚐', '✙', '↙', '̾', '℘', '✷', '⍺', '❌', '⊢', '▵', '✅', 'ⓖ', '☨', '▰', '╡', 'ⓜ', '☤', '∽', '╘', \n '˹', '↨', '♙', '⬇', '♱', '⌡', '⠀', '╛', '❕', '┉', 'ⓟ', '̀', '♖', 'ⓚ', '┆', '⎜', '◜', '⚾', '⤴', '✇', '╟', '⎛', \n '☩', '➲', '➟', 'ⓥ', 'ⓗ', '⏝', '◃', '╢', '↯', '✆', '˃', '⍴', '❇', '⚽', '╒', '̸', '♜', '☓', '➳', '⇄', '☬', '⚑', \n '✐', '⌃', '◅', '▢', '❐', '∊', '☈', '॥', '⎮', '▩', 'ு', '⊹', '‵', '␔', '☊', '➸', '̌', '☿', '⇉', '⊳', '╙', 'ⓦ', \n '⇣', '｛', '̄', '↝', '⎟', '▍', '❗', '״', '΄', '▞', '◁', '⛄', '⇝', '⎪', '♁', '⇠', '☇', '✊', 'ி', '｝', '⭕', '➘', \n '⁀', '☙', '❛', '❓', '⟲', '⇀', '≲', 'ⓕ', '⎥', '\\u06dd', 'ͤ', '₋', '̱', '̎', '♝', '≳', '▙', '➭', '܀', 'ⓖ', '⇛', '▊', \n '⇗', '̷', '⇱', '℅', 'ⓧ', '⚛', '̐', '̕', '⇌', '␀', '≌', 'ⓦ', '⊤', '̓', '☦', 'ⓕ', '▜', '➙', 'ⓨ', '⌨', '◮', '☷', \n '◍', 'ⓚ', '≔', '⏩', '⍳', '℞', '┋', '˻', '▚', '≺', 'ْ', '▟', '➻', '̪', '⏪', '̉', '⎞', '┇', '⍟', '⇪', '▎', '⇦', '␝', \n '⤷', '≖', '⟶', '♗', '̴', '♄', 'ͨ', '̈', '❜', '̡', '▛', '✁', '➩', 'ா', '˂', '↥', '⏎', '⎷', '̲', '➖', '↲', '⩵', '̗', '❢', \n '≎', '⚔', '⇇', '̑', '⊿', '̖', '☍', '➹', '⥊', '⁁', '✢']\n\ndef clean_punct(x):\n for punct in puncts:\n if punct in x:\n x = x.replace(punct, '{}' .format(punct))\n return x","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:14.313879Z","iopub.execute_input":"2021-06-10T17:55:14.314240Z","iopub.status.idle":"2021-06-10T17:55:14.348789Z","shell.execute_reply.started":"2021-06-10T17:55:14.314204Z","shell.execute_reply":"2021-06-10T17:55:14.347757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Remove number**","metadata":{}},{"cell_type":"code","source":"def clean_numbers(x):\n if bool(re.search(r'\\d', x)):\n x = re.sub('[0-9]{5,}', '#####', x)\n x = re.sub('[0-9]{4}', '####', x)\n x = re.sub('[0-9]{3}', '###', x)\n x = re.sub('[0-9]{2}', '##', x)\n return x","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:14.350034Z","iopub.execute_input":"2021-06-10T17:55:14.350498Z","iopub.status.idle":"2021-06-10T17:55:14.371819Z","shell.execute_reply.started":"2021-06-10T17:55:14.350463Z","shell.execute_reply":"2021-06-10T17:55:14.371022Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" **Create a vector of mispell words**
\n **Convert the shortened form to the original**","metadata":{}},{"cell_type":"code","source":"mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'bitcoin', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', \n 'electroneum':'bitcoin','nanodegree':'degree','hotstar':'star','dream11':'dream','ftre':'fire','tensorflow':'framework','unocoin':'bitcoin',\n 'lnmiit':'limit','unacademy':'academy','altcoin':'bitcoin','altcoins':'bitcoin','litecoin':'bitcoin','coinbase':'bitcoin','cryptocurency':'cryptocurrency',\n 'simpliv':'simple','quoras':'quora','schizoids':'psychopath','remainers':'remainder','twinflame':'soulmate','quorans':'quora','brexit':'demonetized',\n 'iiest':'institute','dceu':'comics','pessat':'exam','uceed':'college','bhakts':'devotee','boruto':'anime',\n 'cryptocoin':'bitcoin','blockchains':'blockchain','fiancee':'fiance','redmi':'smartphone','oneplus':'smartphone','qoura':'quora','deepmind':'framework','ryzen':'cpu','whattsapp':'whatsapp',\n 'undertale':'adventure','zenfone':'smartphone','cryptocurencies':'cryptocurrencies','koinex':'bitcoin','zebpay':'bitcoin','binance':'bitcoin','whtsapp':'whatsapp',\n 'reactjs':'framework','bittrex':'bitcoin','bitconnect':'bitcoin','bitfinex':'bitcoin','yourquote':'your quote','whyis':'why is','jiophone':'smartphone',\n 'dogecoin':'bitcoin','onecoin':'bitcoin','poloniex':'bitcoin','7700k':'cpu','angular2':'framework','segwit2x':'bitcoin','hashflare':'bitcoin','940mx':'gpu',\n 'openai':'framework','hashflare':'bitcoin','1050ti':'gpu','nearbuy':'near buy','freebitco':'bitcoin','antminer':'bitcoin','filecoin':'bitcoin','whatapp':'whatsapp',\n 'empowr':'empower','1080ti':'gpu','crytocurrency':'cryptocurrency','8700k':'cpu','whatsaap':'whatsapp','g4560':'cpu','payymoney':'pay money',\n 'fuckboys':'fuck boys','intenship':'internship','zcash':'bitcoin','demonatisation':'demonetization','narcicist':'narcissist','mastuburation':'masturbation',\n 'trignometric':'trigonometric','cryptocurreny':'cryptocurrency','howdid':'how did','crytocurrencies':'cryptocurrencies','phycopath':'psychopath',\n 'bytecoin':'bitcoin','possesiveness':'possessiveness','scollege':'college','humanties':'humanities','altacoin':'bitcoin','demonitised':'demonetized',\n 'brasília':'brazilia','accolite':'accolyte','econimics':'economics','varrier':'warrier','quroa':'quora','statergy':'strategy','langague':'language',\n 'splatoon':'game','7600k':'cpu','gate2018':'gate 2018','in2018':'in 2018','narcassist':'narcissist','jiocoin':'bitcoin','hnlu':'hulu','7300hq':'cpu',\n 'weatern':'western','interledger':'blockchain','deplation':'deflation', 'cryptocurrencies':'cryptocurrency', 'bitcoin':'blockchain cryptocurrency',}\n\ndef _get_mispell(mispell_dict):\n mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n def replace(match):\n return mispellings[match.group(0)]\n return mispellings_re.sub(replace, text)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:14.373221Z","iopub.execute_input":"2021-06-10T17:55:14.373582Z","iopub.status.idle":"2021-06-10T17:55:14.397834Z","shell.execute_reply.started":"2021-06-10T17:55:14.373547Z","shell.execute_reply":"2021-06-10T17:55:14.397167Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Convert abbreviated words**","metadata":{}},{"cell_type":"code","source":"contraction_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\", \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\", \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\", \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\", \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\"}\n\ndef _get_contractions(contraction_dict):\n contraction_re = re.compile('(%s)' % '|'.join(contraction_dict.keys()))\n return contraction_dict, contraction_re\n\ncontractions, contractions_re = _get_contractions(contraction_dict)\n\ndef replace_contractions(text):\n def replace(match):\n return contractions[match.group(0)]\n return contractions_re.sub(replace, text)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:14.399376Z","iopub.execute_input":"2021-06-10T17:55:14.399766Z","iopub.status.idle":"2021-06-10T17:55:14.417803Z","shell.execute_reply.started":"2021-06-10T17:55:14.399729Z","shell.execute_reply":"2021-06-10T17:55:14.417009Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **In order to process we must clean text**","metadata":{}},{"cell_type":"markdown","source":" **Remove stopwords**","metadata":{}},{"cell_type":"code","source":"stopword_list = nltk.corpus.stopwords.words('english')\ndef remove_stopwords(text, is_lower_case=True):\n tokenizer = ToktokTokenizer()\n tokens = tokenizer.tokenize(text)\n tokens = [token.strip() for token in tokens]\n if is_lower_case:\n filtered_tokens = [token for token in tokens if token not in stopword_list]\n else:\n filtered_tokens = [token for token in tokens if token.lower() not in stopword_list]\n filtered_text = ' '.join(filtered_tokens)\n return filtered_text","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:14.418919Z","iopub.execute_input":"2021-06-10T17:55:14.419611Z","iopub.status.idle":"2021-06-10T17:55:14.442826Z","shell.execute_reply.started":"2021-06-10T17:55:14.419576Z","shell.execute_reply":"2021-06-10T17:55:14.442173Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" **Convert words with the same variation of a word into a single word**","metadata":{}},{"cell_type":"code","source":"from nltk.stem import SnowballStemmer\nfrom nltk.tokenize.toktok import ToktokTokenizer\ndef stem_text(text):\n tokenizer = ToktokTokenizer()\n stemmer = SnowballStemmer('english')\n tokens = tokenizer.tokenize(text)\n tokens = [token.strip() for token in tokens]\n tokens = [stemmer.stem(token) for token in tokens]\n return ' '.join(tokens)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:14.444049Z","iopub.execute_input":"2021-06-10T17:55:14.444386Z","iopub.status.idle":"2021-06-10T17:55:14.450424Z","shell.execute_reply.started":"2021-06-10T17:55:14.444354Z","shell.execute_reply":"2021-06-10T17:55:14.449645Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"from nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize.toktok import ToktokTokenizer\nwordnet_lemmatizer = WordNetLemmatizer()\ndef lemma_text(text):\n tokenizer = ToktokTokenizer()\n tokens = tokenizer.tokenize(text)\n tokens = [token.strip() for token in tokens]\n tokens = [wordnet_lemmatizer.lemmatize(token) for token in tokens]\n return ' '.join(tokens)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:14.451766Z","iopub.execute_input":"2021-06-10T17:55:14.452362Z","iopub.status.idle":"2021-06-10T17:55:14.464267Z","shell.execute_reply.started":"2021-06-10T17:55:14.452325Z","shell.execute_reply":"2021-06-10T17:55:14.463296Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":" **Clean sentences by using all above features**","metadata":{}},{"cell_type":"code","source":"def clean_sentence(x):\n x = x.lower()\n x = clean_punct(x)\n x = clean_numbers(x)\n x = replace_typical_misspell(x)\n x = remove_stopwords(x)\n x = replace_contractions(x)\n x = stem_text(x)\n x = lemma_text(x)\n x = x.replace(\"'\",\"\")\n return x","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:14.465238Z","iopub.execute_input":"2021-06-10T17:55:14.465467Z","iopub.status.idle":"2021-06-10T17:55:14.475235Z","shell.execute_reply.started":"2021-06-10T17:55:14.465446Z","shell.execute_reply":"2021-06-10T17:55:14.474272Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out some sentences after cleaning**","metadata":{}},{"cell_type":"code","source":"train_data['preprocessed_question_text'] = train_data['question_text'].apply(lambda x: clean_sentence(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T17:55:14.476126Z","iopub.execute_input":"2021-06-10T17:55:14.476352Z","iopub.status.idle":"2021-06-10T18:04:24.710663Z","shell.execute_reply.started":"2021-06-10T17:55:14.476331Z","shell.execute_reply":"2021-06-10T18:04:24.709745Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out some sentences of train data after cleaning**","metadata":{}},{"cell_type":"code","source":"train_data.preprocessed_question_text.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:04:24.712055Z","iopub.execute_input":"2021-06-10T18:04:24.712436Z","iopub.status.idle":"2021-06-10T18:04:24.718634Z","shell.execute_reply.started":"2021-06-10T18:04:24.712397Z","shell.execute_reply":"2021-06-10T18:04:24.717849Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data['preprocessed_question_text'] = test_data['question_text'].apply(lambda x: clean_sentence(x))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:04:24.722852Z","iopub.execute_input":"2021-06-10T18:04:24.723345Z","iopub.status.idle":"2021-06-10T18:07:01.088756Z","shell.execute_reply.started":"2021-06-10T18:04:24.723305Z","shell.execute_reply":"2021-06-10T18:07:01.087859Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out some sentences of test data after cleaning**","metadata":{}},{"cell_type":"code","source":"test_data.preprocessed_question_text.head()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:07:01.090660Z","iopub.execute_input":"2021-06-10T18:07:01.091037Z","iopub.status.idle":"2021-06-10T18:07:01.101417Z","shell.execute_reply.started":"2021-06-10T18:07:01.090999Z","shell.execute_reply":"2021-06-10T18:07:01.100608Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **A tag cloud: A novelty visual representation of text data to visualize free form text**","metadata":{}},{"cell_type":"code","source":"def cloud(text, title, size = (10,7)):\n # Processing Text\n words_list = text.unique().tolist()\n words = ' '.join(words_list)\n \n wordcloud = WordCloud(width=800, height=400,\n collocations=False\n ).generate(words)\n \n # Output Visualization\n fig = plt.figure(figsize=size, dpi=80, facecolor='k',edgecolor='k')\n plt.imshow(wordcloud,interpolation='bilinear')\n plt.axis('off')\n plt.title(title, fontsize=25,color='w')\n plt.tight_layout(pad=0)\n plt.show()","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:07:01.102764Z","iopub.execute_input":"2021-06-10T18:07:01.103157Z","iopub.status.idle":"2021-06-10T18:07:01.110607Z","shell.execute_reply.started":"2021-06-10T18:07:01.103121Z","shell.execute_reply":"2021-06-10T18:07:01.109511Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out the visualization of words which appear in sincere questions (train.csv)**","metadata":{}},{"cell_type":"code","source":"cloud(train_data[train_data['target']==0]['question_text'], 'Sincere Questions On question_text')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:07:01.111843Z","iopub.execute_input":"2021-06-10T18:07:01.112401Z","iopub.status.idle":"2021-06-10T18:07:21.872718Z","shell.execute_reply.started":"2021-06-10T18:07:01.112364Z","shell.execute_reply":"2021-06-10T18:07:21.871910Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out the visualization of words which appear in sincere questions (train.csv) AFTER cleaning**","metadata":{}},{"cell_type":"code","source":"cloud(train_data[train_data['target']==0]['preprocessed_question_text'], 'Sincere Questions On preprocessed_question_text')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:07:21.873816Z","iopub.execute_input":"2021-06-10T18:07:21.874191Z","iopub.status.idle":"2021-06-10T18:07:34.823959Z","shell.execute_reply.started":"2021-06-10T18:07:21.874152Z","shell.execute_reply":"2021-06-10T18:07:34.823128Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out the visualization of words which appear in insincere questions (train.csv)**","metadata":{}},{"cell_type":"code","source":"cloud(train_data[train_data['target']==1]['question_text'], 'Insincere Questions On question_text')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:07:34.825295Z","iopub.execute_input":"2021-06-10T18:07:34.825647Z","iopub.status.idle":"2021-06-10T18:07:37.482552Z","shell.execute_reply.started":"2021-06-10T18:07:34.825610Z","shell.execute_reply":"2021-06-10T18:07:37.481765Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Print out the visualization of words which appear in insincere questions (train.csv) AFTER cleaning**","metadata":{}},{"cell_type":"code","source":"cloud(train_data[train_data['target']==1]['preprocessed_question_text'], 'Insincere Questions On preprocessed_question_text')","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:07:37.483890Z","iopub.execute_input":"2021-06-10T18:07:37.484269Z","iopub.status.idle":"2021-06-10T18:07:39.505463Z","shell.execute_reply.started":"2021-06-10T18:07:37.484231Z","shell.execute_reply":"2021-06-10T18:07:39.504649Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **NOTE**\n**+ Insincere questions often have a lot of bad meaning words**
\n**+ However, some words that do not carry a bad meaning have a high frequency like \"people\", \"women\", \"Trump\",... These words belong to stopwords, that is, words that are necessary in grammar but do not give much meaning when viewed individually**","metadata":{}},{"cell_type":"markdown","source":"-----------------------------------------------------------------------------------------------------","metadata":{}},{"cell_type":"markdown","source":"# **Apply GRU Kerax to start training**","metadata":{}},{"cell_type":"markdown","source":"**Import necessary libraries**","metadata":{}},{"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.compat.v1.keras.layers import CuDNNGRU\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:07:39.506769Z","iopub.execute_input":"2021-06-10T18:07:39.507147Z","iopub.status.idle":"2021-06-10T18:07:44.149772Z","shell.execute_reply.started":"2021-06-10T18:07:39.507110Z","shell.execute_reply":"2021-06-10T18:07:44.148980Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **First, we process a bit of data used for training**\n**+ Split the train file into 2 parts: train and validate. The train file will be used for training and the validate file will be used to check if the model is good or not**
\n**+ Fill in \"na\" in the missing data to avoid loss**","metadata":{}},{"cell_type":"code","source":"## split to train and val\ntrain_data, val_data = train_test_split(train_data, test_size=0.1, random_state=2018) #use 10% of train data to validate\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill in \"na\" the missing values\ntrain_X = train_data[\"question_text\"].fillna(\"_na_\").values\nval_X = val_data[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_data[\"question_text\"].fillna(\"_na_\").values","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:07:44.151118Z","iopub.execute_input":"2021-06-10T18:07:44.151450Z","iopub.status.idle":"2021-06-10T18:07:45.271744Z","shell.execute_reply.started":"2021-06-10T18:07:44.151417Z","shell.execute_reply":"2021-06-10T18:07:45.270917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **If we leave the data as strings, the machine will not understand it. We would think of encoding each word as a unique positive integer**\n**E.g: If a sentence consist 10 words, we encode it into a vector of 10x1**","metadata":{}},{"cell_type":"markdown","source":"# **We use Tokenizer to do this. It will encode words into unique positive integers. The lower the number, the more common the word is in the dictionary**\n**To synchronize the data, we also use pad_sequences to ensure that the sentences are all 100 words long:**
\n**+ Cut sentences longer than 100 words**
\n**+ Fill in 0 for enough sentences with less than 100 words. Then each sentence will be represented by a vector number 100x1**","metadata":{}},{"cell_type":"code","source":"## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences into 100 words\ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:07:45.273137Z","iopub.execute_input":"2021-06-10T18:07:45.273616Z","iopub.status.idle":"2021-06-10T18:08:43.279706Z","shell.execute_reply.started":"2021-06-10T18:07:45.273580Z","shell.execute_reply":"2021-06-10T18:08:43.278870Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **We have the set train_X, val_X, test_X which are the number vectors corresponding to each question in the files**","metadata":{}},{"cell_type":"markdown","source":"# **Observe questions after encoding to 0**","metadata":{}},{"cell_type":"code","source":"print(train_X[0])","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:08:43.281071Z","iopub.execute_input":"2021-06-10T18:08:43.281403Z","iopub.status.idle":"2021-06-10T18:08:43.287160Z","shell.execute_reply.started":"2021-06-10T18:08:43.281369Z","shell.execute_reply":"2021-06-10T18:08:43.286262Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Get the target column of the train file and the validate file for training**","metadata":{}},{"cell_type":"code","source":"## Get the target values\ntrain_y = train_data['target'].values\nval_y = val_data['target'].values","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:08:43.288420Z","iopub.execute_input":"2021-06-10T18:08:43.289029Z","iopub.status.idle":"2021-06-10T18:08:43.300891Z","shell.execute_reply.started":"2021-06-10T18:08:43.288994Z","shell.execute_reply":"2021-06-10T18:08:43.300017Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Build GRU model**","metadata":{}},{"cell_type":"markdown","source":"**+ Input will be the string vectors corresponding to the questions**
\n**+ A string will have 100 words corresponding to a 100-dimensional vector**
\n**+ Embedding will help the machine learn what the words mean. Embedding will convert each word into a 1x300 vector representing the meaning of that word, which is a sentence will be a vector of numbers 100x300**
\n**+ The Bidirection layer will help the machine learn the meaning of each sentence based on the order of words on the neural network**
\n**+ For each of the 128 features, the global layer chooses the word with the best feature**
\n**--> The remaining in the model are used for classification**","metadata":{}},{"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(LSTM(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nprint(model.summary())","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:08:43.302092Z","iopub.execute_input":"2021-06-10T18:08:43.302544Z","iopub.status.idle":"2021-06-10T18:08:45.811616Z","shell.execute_reply.started":"2021-06-10T18:08:43.302508Z","shell.execute_reply":"2021-06-10T18:08:45.810660Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **Start training with the file train_X train_Y** \n# **Feed the data into the neural network twice**\n# **Each time subdivided into batch_size is 512 sentences. The data used for testing is val_X and val_y**","metadata":{}},{"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:08:45.812972Z","iopub.execute_input":"2021-06-10T18:08:45.813325Z","iopub.status.idle":"2021-06-10T18:29:33.078609Z","shell.execute_reply.started":"2021-06-10T18:08:45.813288Z","shell.execute_reply":"2021-06-10T18:29:33.077834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **The model works quite well. For a more detailed look, calculate the model's F1-score with thresholds from 0.1 to 0.5**","metadata":{}},{"cell_type":"code","source":"pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n thresh = np.round(thresh, 2)\n print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_noemb_val_y>thresh).astype(int))))","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:29:33.079970Z","iopub.execute_input":"2021-06-10T18:29:33.080339Z","iopub.status.idle":"2021-06-10T18:29:39.820520Z","shell.execute_reply.started":"2021-06-10T18:29:33.080301Z","shell.execute_reply":"2021-06-10T18:29:39.819530Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **We see that the F1-score in the threshold range of 0.26-0.4 is quite good**\n# **We need to be more concerned with \"missing is better than mistaken\", so we should consider threshold < 0.5 (closer to 0 than 1)**\n# **--> We would rather miss identifying insincere questions than misidentifying a insincere question as sincere**","metadata":{}},{"cell_type":"code","source":"pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1)","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:29:39.821818Z","iopub.execute_input":"2021-06-10T18:29:39.822190Z","iopub.status.idle":"2021-06-10T18:29:54.247306Z","shell.execute_reply.started":"2021-06-10T18:29:39.822151Z","shell.execute_reply":"2021-06-10T18:29:54.246507Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# **After training, save and submit**","metadata":{}},{"cell_type":"code","source":"pred_noemb_test_y = (pred_noemb_test_y > 0.33).astype(int)\nout_df = pd.DataFrame({\"qid\":test_data[\"qid\"].values})\nout_df['prediction'] = pred_noemb_test_y\nout_df.to_csv(\"submission.csv\", index=False)\nprint('Successfully saved submission')\npred_noemb_test_y","metadata":{"execution":{"iopub.status.busy":"2021-06-10T18:29:54.251091Z","iopub.execute_input":"2021-06-10T18:29:54.253026Z","iopub.status.idle":"2021-06-10T18:29:55.306584Z","shell.execute_reply.started":"2021-06-10T18:29:54.252986Z","shell.execute_reply":"2021-06-10T18:29:55.305794Z"},"trusted":true},"execution_count":null,"outputs":[]}]}