{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"","metadata":{}},{"cell_type":"markdown","source":"

Homework 3 - Building NLP Pipeline

\n
Problem : Quora Insincere Questions Classification -\nDetect toxic content to improve online conversations.
\n
Details : We are given a dataset of word sequences with target labels 1(insincere) and 0(sincere). Our goal is to train nlp model with neural networks which will have decent accuracy. Word embeddings are also provided.
","metadata":{}},{"cell_type":"markdown","source":"

Imports

","metadata":{}},{"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nfrom tqdm import tqdm\ntqdm.pandas()\nimport re\nimport matplotlib.pyplot as plt\nimport plotly.graph_objs as go\nfrom plotly import subplots\nimport plotly.offline as py\nfrom wordcloud import WordCloud, STOPWORDS\nfrom collections import defaultdict\nimport operator \nimport os\nfrom torchtext.data.utils import get_tokenizer\nfrom torchtext.vocab import build_vocab_from_iterator\nfrom torchtext.data.utils import get_tokenizer\nimport torch\nimport torch.nn as nn\nfrom torchtext.legacy import data\nimport itertools\nimport matplotlib.pyplot as plt\nfrom sklearn.metrics import confusion_matrix\nimport torch.nn.functional as F\nimport zipfile","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:36.376694Z","iopub.execute_input":"2022-05-07T16:36:36.377205Z","iopub.status.idle":"2022-05-07T16:36:38.79402Z","shell.execute_reply.started":"2022-05-07T16:36:36.377156Z","shell.execute_reply":"2022-05-07T16:36:38.792994Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Part 1: Read and Analyse Data

","metadata":{}},{"cell_type":"code","source":"train_df = pd.read_csv(\"../input/quora-insincere-questions-classification/train.csv\")\ntest_df = pd.read_csv(\"../input/quora-insincere-questions-classification/test.csv\")","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:38.795528Z","iopub.execute_input":"2022-05-07T16:36:38.795753Z","iopub.status.idle":"2022-05-07T16:36:44.605713Z","shell.execute_reply.started":"2022-05-07T16:36:38.795727Z","shell.execute_reply":"2022-05-07T16:36:44.60474Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Train data dimension: ', train_df.shape)\ntrain_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:44.606945Z","iopub.execute_input":"2022-05-07T16:36:44.607196Z","iopub.status.idle":"2022-05-07T16:36:44.630544Z","shell.execute_reply.started":"2022-05-07T16:36:44.607165Z","shell.execute_reply":"2022-05-07T16:36:44.629564Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we can see we are dealing with sequential input which has one label output, therefore we are going to use rnn sentiment analys technique which fits best to this problem(According to my current knowledge of NLP :D).","metadata":{}},{"cell_type":"code","source":"train_df.isna().sum()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:44.633175Z","iopub.execute_input":"2022-05-07T16:36:44.633516Z","iopub.status.idle":"2022-05-07T16:36:44.931195Z","shell.execute_reply.started":"2022-05-07T16:36:44.633455Z","shell.execute_reply":"2022-05-07T16:36:44.930033Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"As we know from the problem description, our sequential inputs are questions, so let's check how many of these inputs has no question symbol in it.","metadata":{}},{"cell_type":"code","source":"not_contains_question_symbols = train_df[~train_df['question_text'].str.contains(pat = \"\\?\")]\nprint(\"Count - \",not_contains_question_symbols[\"question_text\"].count())\nprint(not_contains_question_symbols[\"question_text\"][:40].to_string())\n\ndel not_contains_question_symbols","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:44.93264Z","iopub.execute_input":"2022-05-07T16:36:44.932965Z","iopub.status.idle":"2022-05-07T16:36:45.885259Z","shell.execute_reply.started":"2022-05-07T16:36:44.932932Z","shell.execute_reply":"2022-05-07T16:36:45.884666Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We got only 300 records here and as it's shown above, most of these records have question format, so we will leave them untouched.","metadata":{}},{"cell_type":"markdown","source":"Let's also check the distribution of sentence lengths to determine the mean one.","metadata":{}},{"cell_type":"code","source":"train_df['word_count'] = train_df['question_text'].apply(lambda x: len(x.split()))\ntrain_df.pivot(columns='target', values='word_count').plot.hist(bins=range(0,60))","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:45.886557Z","iopub.execute_input":"2022-05-07T16:36:45.886943Z","iopub.status.idle":"2022-05-07T16:36:49.34365Z","shell.execute_reply.started":"2022-05-07T16:36:45.886903Z","shell.execute_reply":"2022-05-07T16:36:49.343014Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"cnt_srs = train_df['target'].value_counts()\nlabels = (np.array(cnt_srs.index))\nsizes = (np.array((cnt_srs / cnt_srs.sum())*100))\n\ntrace = go.Pie(labels=labels, values=sizes)\nlayout = go.Layout(\n title='Target distribution',\n font=dict(size=18),\n width=600,\n height=600,\n)\ndata = [trace]\nfig = go.Figure(data=data, layout=layout)\npy.iplot(fig, filename=\"usertype\")\n\nprint(\"Label 1 count: \", train_df[train_df['target'] == 1].count()[0])\nprint(\"Label 0 count: \", train_df[train_df['target'] == 0].count()[0])\n\ndel cnt_srs, labels, sizes, trace, layout, data, fig","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:49.344715Z","iopub.execute_input":"2022-05-07T16:36:49.345171Z","iopub.status.idle":"2022-05-07T16:36:50.712772Z","shell.execute_reply.started":"2022-05-07T16:36:49.345137Z","shell.execute_reply":"2022-05-07T16:36:50.711618Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Unfortunately given data has very unbalanced distribution of targets, target 0 has way more observed data...","metadata":{}},{"cell_type":"code","source":"max_seq_len = round(sum(train_df['word_count'])/len(train_df))\nprint('Average word count in questions is ', max_seq_len)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:50.714413Z","iopub.execute_input":"2022-05-07T16:36:50.714813Z","iopub.status.idle":"2022-05-07T16:36:50.891532Z","shell.execute_reply.started":"2022-05-07T16:36:50.714762Z","shell.execute_reply":"2022-05-07T16:36:50.890809Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will pad or trim all sequences to **max_seq_len** length later in data processing stage.","metadata":{}},{"cell_type":"code","source":"print('Max word length of questions in train is {0:.0f}.'.format(np.max(train_df['question_text'].apply(lambda x: len(x.split())))))","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:50.892653Z","iopub.execute_input":"2022-05-07T16:36:50.893299Z","iopub.status.idle":"2022-05-07T16:36:52.80473Z","shell.execute_reply.started":"2022-05-07T16:36:50.893257Z","shell.execute_reply":"2022-05-07T16:36:52.803714Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Word Cloud of Question Words

","metadata":{}},{"cell_type":"code","source":"def plot_wordcloud(text, mask=None, max_words=200, max_font_size=100, figure_size=(24.0,16.0), \n title = None, title_size=40, image_color=False):\n stopwords = set(STOPWORDS)\n more_stopwords = {'one', 'br', 'Po', 'th', 'sayi', 'fo', 'Unknown'}\n stopwords = stopwords.union(more_stopwords)\n\n wordcloud = WordCloud(background_color='black',\n stopwords = stopwords,\n max_words = max_words,\n max_font_size = max_font_size, \n random_state = 42,\n width=800, \n height=400,\n mask = mask)\n wordcloud.generate(str(text))\n \n plt.figure(figsize=figure_size)\n if image_color:\n image_colors = ImageColorGenerator(mask);\n plt.imshow(wordcloud.recolor(color_func=image_colors), interpolation=\"bilinear\");\n plt.title(title, fontdict={'size': title_size, \n 'verticalalignment': 'bottom'})\n else:\n plt.imshow(wordcloud);\n plt.title(title, fontdict={'size': title_size, 'color': 'black', \n 'verticalalignment': 'bottom'})\n plt.axis('off');\n plt.tight_layout() \n \n del stopwords, more_stopwords, wordcloud\n \nplot_wordcloud(train_df[\"question_text\"], title=\"Word Cloud of Questions\")","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:36:52.807582Z","iopub.execute_input":"2022-05-07T16:36:52.807854Z","iopub.status.idle":"2022-05-07T16:36:53.726421Z","shell.execute_reply.started":"2022-05-07T16:36:52.80782Z","shell.execute_reply":"2022-05-07T16:36:53.72541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Word Frequency plot of sincere and insincere questions

","metadata":{}},{"cell_type":"code","source":"train1_df = train_df[train_df[\"target\"]==1]\ntrain0_df = train_df[train_df[\"target\"]==0]\n\n## custom function for ngram generation ##\ndef generate_ngrams(text, n_gram=1):\n token = [token for token in text.lower().split(\" \") if token != \"\" if token not in STOPWORDS]\n ngrams = zip(*[token[i:] for i in range(n_gram)])\n return [\" \".join(ngram) for ngram in ngrams]\n\n## custom function for horizontal bar chart ##\ndef horizontal_bar_chart(df, color):\n trace = go.Bar(\n y=df[\"word\"].values[::-1],\n x=df[\"wordcount\"].values[::-1],\n showlegend=False,\n orientation = 'h',\n marker=dict(\n color=color,\n ),\n )\n return trace\n\n## Get the bar chart from sincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train0_df[\"question_text\"]:\n for word in generate_ngrams(sent):\n freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace0 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n## Get the bar chart from insincere questions ##\nfreq_dict = defaultdict(int)\nfor sent in train1_df[\"question_text\"]:\n for word in generate_ngrams(sent):\n freq_dict[word] += 1\nfd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\nfd_sorted.columns = [\"word\", \"wordcount\"]\ntrace1 = horizontal_bar_chart(fd_sorted.head(50), 'blue')\n\n# Creating two subplots\nfig = subplots.make_subplots(rows=1, cols=2, vertical_spacing=0.04,\n subplot_titles=[\"Frequent words of sincere questions\", \n \"Frequent words of insincere questions\"])\nfig.append_trace(trace0, 1, 1)\nfig.append_trace(trace1, 1, 2)\nfig['layout'].update(height=1200, width=900, paper_bgcolor='rgb(233,233,233)', title=\"Word Count Plots\")\npy.iplot(fig, filename='word-plots')\n\ndel train1_df, train0_df, freq_dict, fd_sorted, trace0, trace1","metadata":{"_kg_hide-input":true,"execution":{"iopub.status.busy":"2022-05-07T16:36:53.727875Z","iopub.execute_input":"2022-05-07T16:36:53.728163Z","iopub.status.idle":"2022-05-07T16:37:06.200168Z","shell.execute_reply.started":"2022-05-07T16:36:53.728126Z","shell.execute_reply":"2022-05-07T16:37:06.199123Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Observations**:\n\n1. Both sides share some top words for example 'will', 'people', 'think'\n2. 'trump' is on the second place of insincere words...\n","metadata":{}},{"cell_type":"markdown","source":"

Part 2: Data Preprocessig

\n

We need to process data to match embeddings dictionary words with high accuracy.

\n

Let's define some helper functions:

","metadata":{}},{"cell_type":"code","source":"def build_vocab(sentences, verbose = True):\n \"\"\"\n :param sentences: list of list of words\n :return: dictionary of words and their count\n \"\"\"\n vocab = {}\n for sentence in tqdm(sentences, disable = (not verbose)):\n for word in sentence:\n try:\n vocab[word] += 1\n except KeyError:\n vocab[word] = 1\n return vocab\n\ndef check_coverage(vocab,embeddings_index):\n a = {}\n oov = {}\n k = 0\n i = 0\n for word in tqdm(vocab):\n try:\n a[word] = embeddings_index[word]\n k += vocab[word]\n except:\n\n oov[word] = vocab[word]\n i += vocab[word]\n pass\n\n print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n print('Found embeddings for {:.2%} of all text'.format(k / (k + i)))\n sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n return sorted_x","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:37:06.202027Z","iopub.execute_input":"2022-05-07T16:37:06.202521Z","iopub.status.idle":"2022-05-07T16:37:06.214275Z","shell.execute_reply.started":"2022-05-07T16:37:06.202454Z","shell.execute_reply":"2022-05-07T16:37:06.213226Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Let's start with fixing common misspellings and making words lowercase.","metadata":{}},{"cell_type":"code","source":"mispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\", \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\", \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\", \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\", \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef _get_mispell(mispell_dict):\n mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n def replace(match):\n return mispellings[match.group(0)]\n return mispellings_re.sub(replace, text)\n\ntrain_df[\"question_text\"] = train_df[\"question_text\"].str.lower()\ntest_df[\"question_text\"] = test_df[\"question_text\"].str.lower()\n\ntrain_df[\"question_text\"] = train_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\ntest_df[\"question_text\"] = test_df[\"question_text\"].apply(lambda x: replace_typical_misspell(x))\n","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:37:06.216128Z","iopub.execute_input":"2022-05-07T16:37:06.216658Z","iopub.status.idle":"2022-05-07T16:38:09.211788Z","shell.execute_reply.started":"2022-05-07T16:37:06.216611Z","shell.execute_reply":"2022-05-07T16:38:09.210824Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, let's handle punctuations.","metadata":{}},{"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•', '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`', '<', '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\ndef clean_text(x):\n x = str(x)\n for punct in \"/-'\":\n x = x.replace(punct, ' ')\n for punct in '&':\n x = x.replace(punct, f' {punct} ')\n for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n x = x.replace(punct, '')\n return x\n\ntrain_df[\"question_text\"] = train_df[\"question_text\"].apply(clean_text)\ntest_df[\"question_text\"] = test_df[\"question_text\"].apply(clean_text)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:38:09.213575Z","iopub.execute_input":"2022-05-07T16:38:09.213916Z","iopub.status.idle":"2022-05-07T16:38:25.100657Z","shell.execute_reply.started":"2022-05-07T16:38:09.21387Z","shell.execute_reply":"2022-05-07T16:38:25.099731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"And we need to deal with numbers also.","metadata":{}},{"cell_type":"code","source":"def clean_numbers(x):\n\n x = re.sub('[0-9]{5,}', '#####', x)\n x = re.sub('[0-9]{4}', '####', x)\n x = re.sub('[0-9]{3}', '###', x)\n x = re.sub('[0-9]{2}', '##', x)\n return x\n\ntrain_df[\"question_text\"] = train_df[\"question_text\"].apply(clean_numbers)\ntest_df[\"question_text\"] = test_df[\"question_text\"].apply(clean_numbers)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:38:25.102059Z","iopub.execute_input":"2022-05-07T16:38:25.102288Z","iopub.status.idle":"2022-05-07T16:38:49.849785Z","shell.execute_reply.started":"2022-05-07T16:38:25.10226Z","shell.execute_reply":"2022-05-07T16:38:49.84873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Building vocabulary

\n","metadata":{}},{"cell_type":"code","source":"def load_embed(file):\n def get_coefs(word,*arr): \n return word, np.asarray(arr, dtype='float32')\n \n embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(file, encoding='latin'))\n \n return embeddings_index","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:38:49.851051Z","iopub.execute_input":"2022-05-07T16:38:49.8513Z","iopub.status.idle":"2022-05-07T16:38:49.857077Z","shell.execute_reply.started":"2022-05-07T16:38:49.851264Z","shell.execute_reply":"2022-05-07T16:38:49.856118Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"vocab = build_vocab(train_df[\"question_text\"].progress_apply(lambda x: x.split()))\nprint(\"We have\", len(vocab), \"unique words!\")","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:38:49.859023Z","iopub.execute_input":"2022-05-07T16:38:49.859383Z","iopub.status.idle":"2022-05-07T16:39:02.654916Z","shell.execute_reply.started":"2022-05-07T16:38:49.859337Z","shell.execute_reply":"2022-05-07T16:39:02.654013Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Extracting embeddings.zip and loading glove embeddings.","metadata":{}},{"cell_type":"code","source":"# os.makedirs('./embeddings')\nwith zipfile.ZipFile('../input/quora-insincere-questions-classification/embeddings.zip', 'r') as zip_ref:\n zip_ref.extract('glove.840B.300d/glove.840B.300d.txt', './embeddings')\nglove = './embeddings/glove.840B.300d/glove.840B.300d.txt'\nword_embeddings = load_embed(glove)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:42:42.908254Z","iopub.execute_input":"2022-05-07T16:42:42.908602Z","iopub.status.idle":"2022-05-07T16:43:38.209548Z","shell.execute_reply.started":"2022-05-07T16:42:42.908569Z","shell.execute_reply":"2022-05-07T16:43:38.207916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"oov = check_coverage(vocab,word_embeddings)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.827768Z","iopub.status.idle":"2022-05-07T16:39:02.829043Z","shell.execute_reply.started":"2022-05-07T16:39:02.828747Z","shell.execute_reply":"2022-05-07T16:39:02.82878Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print(len(oov))","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.830324Z","iopub.status.idle":"2022-05-07T16:39:02.831141Z","shell.execute_reply.started":"2022-05-07T16:39:02.830839Z","shell.execute_reply":"2022-05-07T16:39:02.830871Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the results we can see that 99% of words in our data has it's embedding and that's a great result. We may assume that words in oov are not very common and can't be handled generally.","metadata":{}},{"cell_type":"code","source":"print(oov[:30])\ndel oov","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.834539Z","iopub.status.idle":"2022-05-07T16:39:02.835041Z","shell.execute_reply.started":"2022-05-07T16:39:02.834777Z","shell.execute_reply":"2022-05-07T16:39:02.834805Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"unknown_embedding = torch.zeros_like(torch.empty(300))\nvocab_embeddings = {'': unknown_embedding}\nfor word in vocab:\n try:\n vocab_embeddings[word] = torch.tensor(word_embeddings[word])\n except :\n vocab_embeddings[word] = unknown_embedding\ndel word_embeddings, vocab","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.836704Z","iopub.status.idle":"2022-05-07T16:39:02.837214Z","shell.execute_reply.started":"2022-05-07T16:39:02.836935Z","shell.execute_reply":"2022-05-07T16:39:02.836964Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Split data to valid and train, pad/trim to maxlen and then batchify it","metadata":{}},{"cell_type":"code","source":"train_df = train_df[['question_text', 'target']]\no_class = train_df.loc[train_df.target == 0, :]\nl_class = train_df.loc[train_df.target == 1, :]\n\nvalid_o = o_class.iloc[:10000, :]\nvalid_l = l_class.iloc[:10000, :]\n\ntrain_o = o_class.iloc[10000:, :]\ntrain_l = l_class.iloc[10000:, :]\n\ntrain = pd.concat([train_o, train_l], axis=0)\nvalid = pd.concat([valid_o, valid_l], axis=0)\n\ndel train_df, o_class, l_class","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.838676Z","iopub.status.idle":"2022-05-07T16:39:02.83916Z","shell.execute_reply.started":"2022-05-07T16:39:02.838893Z","shell.execute_reply":"2022-05-07T16:39:02.838919Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = list(train.to_records(index=False))\nvalid = list(valid.to_records(index=False))","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.840898Z","iopub.status.idle":"2022-05-07T16:39:02.841387Z","shell.execute_reply.started":"2022-05-07T16:39:02.841114Z","shell.execute_reply":"2022-05-07T16:39:02.84115Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def collate_batch(batch):\n label_list, text_list = [], []\n for text, label in batch:\n label_list.append(label)\n splitted = text.split()\n if max_seq_len < len(splitted):\n splitted = splitted[:max_seq_len]\n elif len(splitted) < max_seq_len:\n splitted += [''] * (max_seq_len - len(splitted))\n text_list.append(torch.stack(list(map(lambda x : vocab_embeddings[x], splitted))))\n label_list = torch.tensor(label_list, dtype=torch.int32)\n text_list = torch.stack(text_list, dim=0)\n return text_list, label_list\n\nfrom torch.utils.data import DataLoader\n\nbatch_size=100\n\ntrain_dl = DataLoader(train, \n batch_size=batch_size, \n shuffle=True, \n collate_fn=collate_batch)\n\nvalid_dl = DataLoader(valid, \n batch_size=batch_size*2, \n shuffle=False, \n collate_fn=collate_batch)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.842754Z","iopub.status.idle":"2022-05-07T16:39:02.843254Z","shell.execute_reply.started":"2022-05-07T16:39:02.842987Z","shell.execute_reply":"2022-05-07T16:39:02.843015Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Part 3: Model Training

\nWe will define our Lstm Rnn Sentiment Analysis Model and train it on preprocessed data.","metadata":{}},{"cell_type":"code","source":"class LSTM(nn.Module):\n def __init__(self, input_size, hidden_size, num_layers, num_classes, dropout=0.05):\n super(LSTM, self).__init__()\n self.num_layers = num_layers\n self.hidden_size = hidden_size\n # We use LSTM becaues it solves vanishing gradient problem which occurs with simple RNN network\n self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)\n self.relu = nn.ReLU()\n self.dropout = nn.Dropout(dropout) # Data is very imbalanced, therefore to avoid overfitting we use dropout technique\n # We need one additional layer for final output\n self.fc = nn.Linear(hidden_size, num_classes)\n\n def forward(self, x):\n h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size)\n c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size)\n _, (hn, cn) = self.lstm(x, (h0,c0))\n \n out = self.relu(hn[1]) # Apply activation function \n out = self.dropout(out) # Dropout Layer\n \n return self.fc(out)\n","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.844528Z","iopub.status.idle":"2022-05-07T16:39:02.844994Z","shell.execute_reply.started":"2022-05-07T16:39:02.844741Z","shell.execute_reply":"2022-05-07T16:39:02.844768Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Confusion Matrix Plot Method**","metadata":{}},{"cell_type":"code","source":"def plot_confusion_matrix(cm, classes,\n normalize=False,\n title='Confusion matrix',\n cmap=plt.cm.Blues):\n \"\"\"\n This function prints and plots the confusion matrix.\n Normalization can be applied by setting `normalize=True`.\n \"\"\"\n \n plt.imshow(cm, interpolation='nearest', cmap=cmap)\n plt.title(title)\n plt.colorbar()\n tick_marks = np.arange(len(classes))\n plt.xticks(tick_marks, classes, rotation=45)\n plt.yticks(tick_marks, classes)\n\n if normalize:\n cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]\n print(\"Normalized confusion matrix\")\n else:\n print('Confusion matrix, without normalization')\n\n thresh = cm.max() / 2.\n for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n plt.text(j, i, cm[i, j],\n horizontalalignment=\"center\",\n color=\"white\" if cm[i, j] > thresh else \"black\")\n\n plt.tight_layout()\n plt.ylabel('True label')\n plt.xlabel('Predicted label')","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.846997Z","iopub.status.idle":"2022-05-07T16:39:02.847518Z","shell.execute_reply.started":"2022-05-07T16:39:02.847239Z","shell.execute_reply":"2022-05-07T16:39:02.847268Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Hyper Parameters:**","metadata":{}},{"cell_type":"code","source":"EMBEDDING_DIM = 300\nHIDDEN_DIM = 128\nSEQUENCE_LEN = max_seq_len\nHIDDEN_LAYERS = 2\n\nNUM_EPOCHS = 3\nLEARNING_RATE = 0.001\nNUM_CLASSES = 2\nBATCH_SIZE = batch_size","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.848785Z","iopub.status.idle":"2022-05-07T16:39:02.849266Z","shell.execute_reply.started":"2022-05-07T16:39:02.848999Z","shell.execute_reply":"2022-05-07T16:39:02.849026Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Model, LossFunction, Optimizer:**","metadata":{}},{"cell_type":"code","source":"model = LSTM(EMBEDDING_DIM, HIDDEN_DIM, HIDDEN_LAYERS, NUM_CLASSES)\nloss_fn = nn.CrossEntropyLoss(reduction='mean')\noptimizer = torch.optim.Adam(model.parameters(), lr=LEARNING_RATE) ","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.851009Z","iopub.status.idle":"2022-05-07T16:39:02.851508Z","shell.execute_reply.started":"2022-05-07T16:39:02.851233Z","shell.execute_reply":"2022-05-07T16:39:02.851259Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Train Loop

","metadata":{}},{"cell_type":"code","source":"def calc_accuracy(model: nn.Module):\n model.eval()\n\n Y, Y_PRED, Y_VAL = [], [], []\n with torch.no_grad():\n for i, (x, y) in enumerate(valid_dl):\n y_pred = model(x)\n\n Y.extend(y.tolist())\n Y_PRED.extend(y_pred.argmax(dim=1).tolist())\n Y_VAL.append(y_pred.cpu())\n\n cnf_matrix = confusion_matrix(Y, Y_PRED)\n\n print('Epoch Valid Loss: ', torch.nn.functional.cross_entropy(torch.cat(Y_VAL, dim=0), torch.LongTensor(Y)).item())\n\n plt.figure(figsize=(5, 5))\n plot_confusion_matrix(cnf_matrix, \n classes=['Neg', 'Pos'],\n title='Confusion Matrix')\n\n plt.show()\n\ndef train():\n for epoch in range(NUM_EPOCHS):\n for i, (x, y) in enumerate(train_dl):\n model.train()\n optimizer.zero_grad()\n y_pred = model(x)\n loss = loss_fn(y_pred, y.long())\n loss.backward()\n optimizer.step()\n\n if (i + 1) % 1000 == 0:\n print(f'Epoch - {epoch + 1}, Iteration - {i + 1}, Train Loss {loss.item()}')\n calc_accuracy(model)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.852656Z","iopub.status.idle":"2022-05-07T16:39:02.853132Z","shell.execute_reply.started":"2022-05-07T16:39:02.852867Z","shell.execute_reply":"2022-05-07T16:39:02.852893Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.854277Z","iopub.status.idle":"2022-05-07T16:39:02.854771Z","shell.execute_reply.started":"2022-05-07T16:39:02.854513Z","shell.execute_reply":"2022-05-07T16:39:02.85454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Sample Predictions**\n","metadata":{}},{"cell_type":"code","source":"def clean(sentence):\n sentence = sentence.lower()\n sentence = replace_typical_misspell(sentence)\n sentence = clean_text(sentence)\n sentence = clean_numbers(sentence)\n return sentence\n\ndef sentenceToEmbeddings(sentence):\n tokens = []\n for token in clean(sentence).split():\n try:\n tokens.append(vocab_embeddings[token])\n except:\n tokens.append(unknown_embedding)\n return torch.stack(tokens).unsqueeze(0)\n\ndef predict_sentiment(sentence):\n try:\n tensor = sentenceToEmbeddings(sentence)\n except:\n return 1 # if sentence is badly formatted or blank, it surely is insincere \n predicted = model(tensor)\n return predicted.argmax(dim=1).item()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.855864Z","iopub.status.idle":"2022-05-07T16:39:02.856334Z","shell.execute_reply.started":"2022-05-07T16:39:02.856073Z","shell.execute_reply":"2022-05-07T16:39:02.856099Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Sincere Prediction**","metadata":{}},{"cell_type":"code","source":"sincere_sentence = 'hello, i am fat. How can i lose some weight?'\nprint(predict_sentiment(sincere_sentence))","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.857606Z","iopub.status.idle":"2022-05-07T16:39:02.858067Z","shell.execute_reply.started":"2022-05-07T16:39:02.857819Z","shell.execute_reply":"2022-05-07T16:39:02.857845Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"**Insincere Prediction**","metadata":{}},{"cell_type":"code","source":"sincere_sentence = 'Fuck you mate!'\nprint(predict_sentiment(sincere_sentence))","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.859111Z","iopub.status.idle":"2022-05-07T16:39:02.859635Z","shell.execute_reply.started":"2022-05-07T16:39:02.859348Z","shell.execute_reply":"2022-05-07T16:39:02.859376Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"

Part 4: Save Submission.csv

","metadata":{}},{"cell_type":"code","source":"test_df.head()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.863989Z","iopub.status.idle":"2022-05-07T16:39:02.864382Z","shell.execute_reply.started":"2022-05-07T16:39:02.864197Z","shell.execute_reply":"2022-05-07T16:39:02.864218Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.count()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.865402Z","iopub.status.idle":"2022-05-07T16:39:02.865791Z","shell.execute_reply.started":"2022-05-07T16:39:02.865593Z","shell.execute_reply":"2022-05-07T16:39:02.865612Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df['prediction'] = test_df['question_text'].apply(predict_sentiment) ","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.867117Z","iopub.status.idle":"2022-05-07T16:39:02.86742Z","shell.execute_reply.started":"2022-05-07T16:39:02.867262Z","shell.execute_reply":"2022-05-07T16:39:02.867278Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df = test_df[['qid', 'prediction']]\ntest_df.groupby('prediction').count()","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.868833Z","iopub.status.idle":"2022-05-07T16:39:02.869151Z","shell.execute_reply.started":"2022-05-07T16:39:02.868988Z","shell.execute_reply":"2022-05-07T16:39:02.869005Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_df.to_csv('submission.csv',index=False)","metadata":{"execution":{"iopub.status.busy":"2022-05-07T16:39:02.870046Z","iopub.status.idle":"2022-05-07T16:39:02.870348Z","shell.execute_reply.started":"2022-05-07T16:39:02.870188Z","shell.execute_reply":"2022-05-07T16:39:02.870204Z"},"trusted":true},"execution_count":null,"outputs":[]}]}