{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport io\nimport warnings\nimport gc\nimport numpy as np\nimport pandas as pd\nimport re\nfrom tqdm import tqdm\nimport matplotlib.pyplot as plt\nfrom nltk.corpus import stopwords\nfrom gensim.models import KeyedVectors\nimport nltk\nfrom sklearn.model_selection import train_test_split\nimport seaborn as sns\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras import backend as K\nfrom keras import layers\nfrom keras.utils.np_utils import to_categorical\nfrom keras import backend as K\nfrom sklearn.preprocessing import OneHotEncoder,LabelEncoder\nfrom keras import models\nfrom keras import regularizers\nfrom sklearn.metrics import precision_score, accuracy_score, recall_score, f1_score\nimport gc","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/quora-insincere-questions-classification/train.csv')\ntest = pd.read_csv('../input/quora-insincere-questions-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pd.options.display.max_colwidth = 150","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_str(string):\n    string = re.sub(r\"[^A-Za-z0-9(),#!?\\'\\`]\", \" \", string)     \n    string = re.sub(r\"\\'s\", \" \\'s\", string) \n    string = re.sub(r\"\\'ve\", \" \\'ve\", string) \n    string = re.sub(r\"n\\'t\", \" n\\'t\", string) \n    string = re.sub(r\"\\'re\", \" \\'re\", string) \n    string = re.sub(r\"\\'d\", \" \\'d\", string) \n    string = re.sub(r\"\\'ll\", \" \\'ll\", string) \n    string = re.sub(r\",\", \" , \", string)\n    string = re.sub(r\"!\", \" ! \", string) \n    string = re.sub(r\"\\(\", \" ( \", string) \n    string = re.sub(r\"\\)\", \" ) \", string) \n    string = re.sub(r\"\\?\", \" ? \", string) \n    string = re.sub(r\"\\s{2,}\", \" \", string)\n    return string.strip()\n\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\npuncts_rem = [ '•', '@', '£', '·', '`', '→', '°', '€', '♥', '←',  '§', 'Â', '█',  'à', '…', \n '★',   '●', 'â', '►',  '¢',  '¬', '░', '¶', '↑', '±', '¿', '▾',  '¦',  '¥', '▓',  \n '▒',  '▼', '▪', '†', '■',  '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅' \n '↓', '、', '│',  '»',  '♪', '╩', '╚', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø',   '‡',  ]\n\npuncts_keep = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '″', '′','<','›',\n          '：','∙', '）', '，','”','“', '（', '—', '‹', '─','_', '{', '}','^','═','×','≤','−','-','’','²','√','½', '³','¼','⊕','~','¹', '‘', '∞','║', '―', '®','©','™',]\n\ndef remove_punct(x):\n    x = str(x)\n    for punct in puncts_rem:\n        x = x.replace(punct,'')\n    return x\n\ndef clean_text2(x):\n    x = str(x)\n    for punct in puncts_keep:\n        x = x.replace(punct, f' {punct} ')\n\ndef clean_text(x):\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\nmispell_dict = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\",\n                \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\",\n                \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\n                \"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\",\n                \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\n                \"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",\n                \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \n                \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\",\n                \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\",\n                \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\n                \"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\",\n                \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\",\n                \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\",\n                \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\",\n                \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \n                \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\",\n                \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \n                \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\",\n                \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\",\n                \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",\n                \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \n                \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\",\n                \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\",\n                \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \n                \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\n                \"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \n                \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\",\n                \"you've\": \"you have\", 'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',\n                'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',\n                'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ',\n                'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do',\n                'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', \n                'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate',\n                \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data',\n                '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what',\n                'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization'}\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispellings, mispellings_re = _get_mispell(mispell_dict)\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def count_char(x):\n    x = str(x)\n    return len(x)\n  \ndef count_char_minus_space(x):\n    x = str(x)\n    return len(x.replace(' ',''))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Removing Misspellings\ntrain['clean_text'] = train['question_text'].apply(lambda x: replace_typical_misspell(x))\ntest['clean_text'] = test['question_text'].apply(lambda x: replace_typical_misspell(x))\n\n#Removing Unwanted Characters\ntrain['clean_text'] = train['clean_text'].apply(lambda x: remove_punct(x))\ntest['clean_text'] = test['clean_text'].apply(lambda x: remove_punct(x))\n\n#Cleaning Numbers\ntrain['clean_text'] = train['clean_text'].apply(lambda x: clean_numbers(x))\ntest['clean_text'] = test['clean_text'].apply(lambda x: clean_numbers(x))\n\n#Removing Capitals\ntrain['clean_text'] = train['clean_text'].apply(lambda x: x.lower())\ntest['clean_text'] = test['clean_text'].apply(lambda x: x.lower())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"  #Seperating all words\ntrain['clean_text'] = train['clean_text'].apply(lambda x: clean_str(x))\ntest['clean_text'] = test['clean_text'].apply(lambda x: clean_str(x))\n\n#Seperating words from remaining punctuations\ntrain['spaced_text'] = train['clean_text'].apply(lambda x: clean_text(x))\ntest['spaced_text'] = test['clean_text'].apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['spaced_text'].loc[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (7,5))\nsns.set(style = 'darkgrid')\nax = sns.countplot(x = 'target', data=train, linewidth = 0.1)\nfor patch in ax.patches :\n        current_width = patch.get_width()\n        new_width = current_width/4\n        diff = current_width - new_width\n\n        # we change the bar width\n        patch.set_width(new_width)\n        patch.set_x(patch.get_x() + diff/2)\nplt.plot()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of words per sentence\ntrain['sent_length'] = train['clean_text'].apply(lambda x: len(x.split()))\ntest['sent_length'] = test['clean_text'].apply(lambda x: len(x.split()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(14,5))\n#fig.tight_layout(pad = 3.0)\nplt.subplot(1,2,1)\nsns.set(style = 'darkgrid')\nax1 = sns.distplot(train['sent_length'], bins = 8, kde = False, color = 'darkcyan')\nplt.yscale('log')\nplt.title('Training Set: Sentence Length')\nplt.xlabel('Word Count')\nplt.ylabel('Number of Sentences')\n\nplt.subplot(1,2,2)\nsns.set(style = 'darkgrid')\nax2 = sns.distplot(test['sent_length'], bins = 8, kde = False, color = 'c')\nplt.yscale('log')\nplt.title('Test Set: Sentence Length')\nplt.xlabel('Word Count')\nplt.ylabel('Number of Sentences')\n\nplt.subplots_adjust(wspace = 0.5)\nplt.plot()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Number of characters per sentence\ntrain['num_char'] = train['clean_text'].apply(lambda x: count_char_minus_space(x))\ntest['num_char'] = test['clean_text'].apply(lambda x: count_char_minus_space(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(14,5))\n#fig.tight_layout(pad = 3.0)\nplt.subplot(1,2,1)\nsns.set(style = 'darkgrid')\nax1 = sns.distplot(train['num_char'], bins = 8, kde = False, color='sandybrown')\nplt.yscale('log')\nplt.title('Training Set: Number of Characters')\nplt.xlabel('Character Count')\nplt.ylabel('Number of Sentences')\n\nplt.subplot(1,2,2)\nsns.set(style = 'darkgrid')\nax2 = sns.distplot(test['num_char'], bins = 8, kde = False,color = 'darkgoldenrod')\nplt.yscale('log')\nplt.title('Test Set: Number of Characters')\nplt.xlabel('Character Count')\nplt.ylabel('Number of Sentences')\n\nplt.subplots_adjust(wspace = 0.5)\nplt.plot()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize = (7,5))\nax = sns.violinplot(x = 'target', y = 'sent_length', data = train)\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sincere = train.loc[train['target']==1]\ninsincere = train.loc[train['target']==0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(14,5))\n#fig.tight_layout(pad = 3.0)\nplt.subplot(1,2,1)\nsns.set(style = 'darkgrid')\nax1 = sns.distplot(sincere['num_char'], bins = 8, kde = False, color='sandybrown')\nplt.yscale('log')\nplt.title('Sincere: Number of Characters')\nplt.xlabel('Character Count')\nplt.ylabel('Number of Sentences')\n\nplt.subplot(1,2,2)\nsns.set(style = 'darkgrid')\nax2 = sns.distplot(insincere['num_char'], bins = 8, kde = False,color = 'darkgoldenrod')\nplt.yscale('log')\nplt.title('Insincere: Number of Characters')\nplt.xlabel('Character Count')\nplt.ylabel('Number of Sentences')\n\nplt.subplots_adjust(wspace = 0.5)\nplt.plot()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Percentage of Sincere Questions: '+ str(len(sincere)) + '/' + str(len(train)))\nprint('Percentage of Sincere Questions: {:.2f}'.format((len(sincere)/len(train))*100))\nprint('Percentage of Insincere Questions: '+ str(len(insincere)) + '/' + str(len(train)))\nprint('Percentage of Insincere Questions: {:.2f}'.format((len(insincere)/len(train))*100))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure(figsize=(14,5))\n#fig.tight_layout(pad = 3.0)\nplt.subplot(1,2,1)\nsns.set(style = 'darkgrid')\nax1 = sns.distplot(sincere['sent_length'], bins = 8, kde = False, color='sandybrown')\nplt.yscale('log')\nplt.title('Sincere: Number of Characters')\nplt.xlabel('Character Count')\nplt.ylabel('Number of Sentences')\n\nplt.subplot(1,2,2)\nsns.set(style = 'darkgrid')\nax2 = sns.distplot(insincere['sent_length'], bins = 8, kde = False,color = 'darkgoldenrod')\nplt.yscale('log')\nplt.title('Insincere: Number of Characters')\nplt.xlabel('Character Count')\nplt.ylabel('Number of Sentences')\n\nplt.subplots_adjust(wspace = 0.5)\nplt.plot()  ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget -P /root/input/ -c \"https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"EMBEDDING_FILE = '/root/input/GoogleNews-vectors-negative300.bin.gz' # from above\nword2vec = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vocab(sentences):\n    vocab = {}\n    for sentence in sentences:\n        for word in sentence:\n            try:\n                vocab[word] += 1\n            except KeyError:\n                   vocab[word] = 1\n    \n    return vocab","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = train[\"spaced_text\"].apply(lambda x: x.split()).values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print( {k + ':' + str(vocab[k]) for k in list(vocab)[:5]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator \n\ndef check_coverage(vocab,embeddings_index):\n  \n    a = {}\n    oov = {}\n    k = 0\n    i = 0\n    for word in tqdm(vocab):\n        try:\n            a[word] = embeddings_index[word]\n            k += vocab[word]\n        except:\n\n            oov[word] = vocab[word]\n            i += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab,word2vec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#removing Numbers/Hashes not having embeddings\ntrain['spaced_text'] = train['spaced_text'].apply(lambda x: re.sub(\"\\S*#+\\S*\",'',str(x)))\ntest['spaced_text'] = train['spaced_text'].apply(lambda x: re.sub(\"\\S*#+\\S*\",'',str(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Cleaning for word2vec\ndef prep_cleaning(text):\n    preposition_removal = ['a','and','of','to']\n    text = str(text)\n    for prep in preposition_removal:\n        text = re.sub('\\s' + prep + '\\s',' ',text)\n  \n    return text\n\ndef remove_unnecessary_punct(text):\n    text = str(text)\n    punctuation = ['?','!','\\.',',','/','\"','$','%','\\'','(',')','*','+','-','/',':',';','<','=','>','@','[','\\\\',']','^','_','`','{','|','}','~','“','”','’']\n    for punct in punctuation:\n        text = text.replace(punct,'')\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['spaced_text'] = train['spaced_text'].apply(lambda x: remove_unnecessary_punct(x))\ntest['spaced_text'] = test['spaced_text'].apply(lambda x: remove_unnecessary_punct(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['spaced_text'] = train['spaced_text'].apply(lambda x: prep_cleaning(x))\ntest['spaced_text'] = test['spaced_text'].apply(lambda x: prep_cleaning(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentences = train[\"spaced_text\"].apply(lambda x: x.split()).values\nvocab = build_vocab(sentences)\noov = check_coverage(vocab,word2vec)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Classification Using Keras"},{"metadata":{"trusted":true},"cell_type":"code","source":"nltk.download('stopwords')\nstop_words = set(stopwords.words('english'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['spaced_text'] = train['spaced_text'].apply(lambda x: list(filter(None,x.split(' '))))\ntest['spaced_text'] = test['spaced_text'].apply(lambda x: list(filter(None,x.split(' '))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['spaced_text'] = train['spaced_text'].apply(lambda x: [word for word in x if word not in stop_words])\ntest['spaced_text'] = test['spaced_text'].apply(lambda x: [word for word in x if word not in stop_words])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['clean_text'] = train['spaced_text'].apply(lambda x: ' '.join(x))\ntest['clean_text'] = test['spaced_text'].apply(lambda x: ' '.join(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Removing Sentences with length 0\nrem_indices = train.loc[(train['spaced_text'].apply(lambda x: len(x))==0)].index.values\ntrain.drop(rem_indices,axis = 0,inplace = True)\ntrain.reset_index(inplace = True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max(train['clean_text'].apply(lambda x: len(x.split(' '))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure()\nsns.distplot(train['spaced_text'].apply(lambda x: len(x)))\nplt.plot()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Gauging the upper limit as {}. No of rows: {}'.format(str(55),str(len(train.loc[train['spaced_text'].apply(lambda x: len(x)) >55]))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"max_features = 20000                                  #Number of most common words we are going to use.\nNB_WORDS = min(max_features, len(vocab))\nVAL_SIZE = 1000\nNB_EPOCHS = 20\nMAX_LENGTH = 55                                         #max(train['clean_text'].apply(lambda x: len(x.split(' ')))) ---> Too much paddding will lead to noise.\nWORD2VEC_DIM = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer(num_words = NB_WORDS,lower = True, split = ' ',oov_token = '<UNK>')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer.fit_on_texts(list(train['spaced_text']))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vocab_size = len(tokenizer.word_index) + 1\nembedding_matrix = np.zeros((max_features, 300))\nfor word, i in tokenizer.word_index.items():\n\tif i < max_features:\n\t\ttry:\n\t\t\tembedding_vector = word2vec[word]\n\t\t\tif embedding_vector is not None:\n\t\t\t\t# words not found in embedding index will be all-zeros.\n\t\t\t\tembedding_matrix[i] = embedding_vector\n\t\texcept KeyError:\n\t\t\tcontinue\n\n## Embedding_martix[0] contains all zeroes as tokenizer index starts from 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['tokenized'] = tokenizer.texts_to_sequences(train['spaced_text'])\nX_test = tokenizer.texts_to_sequences(test['spaced_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train,X_val,Y_train, Y_val = train_test_split(train['tokenized'],train['target'], test_size = 0.1, random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test = tokenizer.texts_to_sequences(test['spaced_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = pad_sequences(X_train, maxlen = MAX_LENGTH, padding = 'post')\nX_val = pad_sequences(X_val, maxlen = MAX_LENGTH, padding = 'post')\nX_test = pad_sequences(X_test, maxlen = MAX_LENGTH, padding = 'post')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import backend as K\n\ndef recall_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef f1_m(y_true, y_pred):\n    precision = precision_m(y_true, y_pred)\n    recall = recall_m(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inp = layers.Input(name = 'inputs', shape =(MAX_LENGTH,))\nx = layers.Embedding(NB_WORDS, WORD2VEC_DIM, weights=[embedding_matrix],trainable = False)(inp)\nx = layers.Flatten()(x)\nx = layers.Dense(10,name = 'FC1', activation = 'relu')(x)\npred = layers.Dense(1, name = 'output_layer',activation = 'sigmoid')(x) ##For K=2 Softmax==Sigmoid.    ##When using Softmax, output nodes == number of classes\nmodel = models.Model(input = inp, output = pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',recall_m,precision_m,f1_m])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X_train, Y_train, batch_size=512, epochs=10, validation_data=(X_val, Y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## CNN Architecture Yoon Kim(2014)"},{"metadata":{"trusted":true},"cell_type":"code","source":"Y = pd.get_dummies(train['target'])\nX_train,X_val,Y_train, Y_val = train_test_split(train['tokenized'],Y, test_size = 0.1, random_state = 42)\nX_test = tokenizer.texts_to_sequences(test['spaced_text'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train = pad_sequences(X_train, maxlen = MAX_LENGTH, padding = 'post')\nX_val = pad_sequences(X_val, maxlen = MAX_LENGTH, padding = 'post')\nX_test = pad_sequences(X_test, maxlen = MAX_LENGTH, padding = 'post')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_filters = 100\nembedding_dim = embedding_matrix.shape[1]\n\ninp_01 = layers.Input(shape = (MAX_LENGTH,))\nembedding_layer = layers.Embedding(NB_WORDS,embedding_dim, weights = [embedding_matrix],input_length = MAX_LENGTH,trainable = False)(inp_01)\n\nconv_01 = layers.Conv1D(filters = num_filters ,kernel_size = 3, activation='relu', kernel_regularizer = tf.keras.regularizers.l2(3) )(embedding_layer)\nconv_02 = layers.Conv1D(filters = num_filters ,kernel_size = 4, activation='relu', kernel_regularizer = tf.keras.regularizers.l2(3) )(embedding_layer)\nconv_03 = layers.Conv1D(filters = num_filters ,kernel_size = 5, activation='relu', kernel_regularizer = tf.keras.regularizers.l2(3) )(embedding_layer)\n\nmax_p01 = layers.MaxPooling1D(pool_size = MAX_LENGTH - 3 + 1)(conv_01)\nmax_p01 = layers.MaxPooling1D(pool_size = MAX_LENGTH - 4 + 1)(conv_02)\nmax_p02 = layers.MaxPooling1D(pool_size = MAX_LENGTH - 5 + 1)(conv_03)\n\nconcatenated = layers.Concatenate(axis = -1)([max_p01, max_p01, max_p02])\n\nflatten = layers.Flatten()(concatenated)\ndropout = layers.Dropout(0.5)(flatten)\n\nCNN_pred_01 = layers.Dense(1, activation = 'sigmoid')(dropout)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CNN_model_01 = models.Model(inp_01, CNN_pred_01)\nCNN_model_01.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',precision_m, recall_m, f1_m])\nCNN_model_01.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_CNN_01 = CNN_model_01.fit(X_train, Y_train, batch_size = 512, epochs = 100, validation_data = (X_val,Y_val))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## LSTM"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Defining the Model\ninp0 = layers.Input(shape= [MAX_LENGTH])\nembedding_layer = layers.Embedding(NB_WORDS,embedding_dim, weights = [embedding_matrix],input_length = MAX_LENGTH,trainable = False)(inp0)\nlstm = layers.LSTM(units = 55)(embedding_layer)\nx = layers.BatchNormalization()(lstm)\ndense = layers.Dense(10, activation = 'relu')(x)\npred = layers.Dense(1, activation = 'sigmoid')(dense)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lstm_model = models.Model(input = inp0, output = pred)\nlstm_model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',recall_m,precision_m,f1_m])\nlstm_model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"history_lstm = lstm_model.fit(X_train, Y_train, batch_size = 512, epochs = 8, validation_data = (X_val,Y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot()\nplt.plot(history_lstm.history['acc'], label = 'Train Accuracy')\nplt.plot(history_lstm.history['val_acc'], label = 'Validation Accuracy')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# CNN with Non-static embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"num_filters = 100\nembedding_dim = embedding_matrix.shape[1]\n\n#Defining the Model\ninp02 = layers.Input(shape = (MAX_LENGTH,))\nembedding_layer = layers.Embedding(NB_WORDS, WORD2VEC_DIM, weights=[embedding_matrix],trainable = True)(inp02)\nconv0 = layers.Conv1D(filters = num_filters, kernel_size = 3, activation='relu', kernel_regularizer = keras.regularizers.l2(3))(embedding_layer)\nconv1 = layers.Conv1D(filters = num_filters, kernel_size = 4, activation='relu', kernel_regularizer = keras.regularizers.l2(3))(embedding_layer)\nconv2 = layers.Conv1D(filters = num_filters, kernel_size = 5, activation='relu', kernel_regularizer = keras.regularizers.l2(3))(embedding_layer)\n\nmax_p0 = layers.MaxPooling1D(pool_size = MAX_LENGTH - 3 + 1)(conv0)\nmax_p1 = layers.MaxPooling1D(pool_size = MAX_LENGTH - 4 + 1)(conv1)\nmax_p2 = layers.MaxPooling1D(pool_size = MAX_LENGTH - 5 + 1)(conv2)\n\n\nconcatenated = layers.Concatenate(axis = -1)([max_p0,max_p1,max_p2])\n\nflatten = layers.Flatten()(concatenated)\ndropout = layers.Dropout(0.1)(flatten)\n\nCNNpred = layers.Dense(1, activation = 'sigmoid')(dropout)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"CNN_model_trainable = models.Model(input = inp02, output = CNNpred)\nCNN_model_trainable.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy',precision_m, recall_m, f1_m])\nCNN_model_trainable.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist_CNN_train = CNN_model_trainable.fit(X_train, Y_train, batch_size = 512, epochs = 100, validation_data = (X_val,Y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}