{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"This is my first NLP problem on Kaggle. I went through a lot of kaggle notebooks and other resources. Mentioning them. Hgave a lot of insightful ideas to do pre-processing.\n\n[](http://www.kaggle.com/theoviel/improve-your-score-with-text-preprocessing-v2)\n\nwww.kaggle.com/theoviel/improve-your-score-with-text-preprocessing-v2\n\nhttps://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings\n\n\nhttp://konukoii.com/blog/2018/02/19/twitter-sentiment-analysis-using-combined-lstm-cnn-models\n\n\nAny suggestion to improve the code highly welcome.\n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.layers import Dense, Input, Embedding, Dropout, Activation,  Conv1D\nfrom tensorflow.keras.layers import GRU,LSTM\nfrom tensorflow.keras.layers import Bidirectional, GlobalMaxPooling1D, MaxPool1D\nfrom tensorflow.keras.models import Model , Sequential\nfrom tensorflow.keras import initializers, regularizers, constraints, optimizers, layers\nfrom tensorflow.keras.callbacks import EarlyStopping","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\ntqdm.pandas()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train=pd.read_csv('/kaggle/input/quora-insincere-questions-classification/train.csv' )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_test=pd.read_csv('/kaggle/input/quora-insincere-questions-classification/test.csv' )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from zipfile import ZipFile\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''embeddings1_index = {}\n\nwith ZipFile('/kaggle/input/quora-insincere-questions-classification/embeddings.zip') as myzip:\n  with myzip.open('glove.840B.300d/glove.840B.300d.txt') as myfile:\n    lines = myfile.readlines()\n    for line in lines:\n      values = line.decode().split(\" \")\n      word = values[0]\n      coefs = np.asarray(values[1:], dtype='float32')\n      embeddings1_index[word] = coefs\n\nprint('Found %s word vectors.' % len(embeddings1_index))\n'''","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import io\nimport zipfile\n\ndim=300\nembeddings1_index={}\n\nwith zipfile.ZipFile(\"../input/quora-insincere-questions-classification/embeddings.zip\") as zf:\n    with io.TextIOWrapper(zf.open(\"glove.840B.300d/glove.840B.300d.txt\"), encoding=\"utf-8\") as f:\n        for line in tqdm(f):\n            values=line.split(' ') # \".split(' ')\" only for glove-840b-300d; for all other files, \".split()\" works\n            word=values[0]\n            vectors=np.asarray(values[1:],'float32')\n            embeddings1_index[word]=vectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del vectors,word,values,line,f,zf,io","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del ZipFile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Found %s word vectors.' % len(embeddings1_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del zipfile","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## Creating the vocabulary of words\ndef build_vocab(sentences,verbose=True):\n\n    vocab={}\n    for sentence in tqdm(sentences,disable= (not verbose)):\n        for word in sentence:\n            try:\n                vocab[word] +=1\n            except:\n                vocab[word] =1\n    return vocab    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"sentences=df_train['question_text'].progress_apply(lambda x : x.split()).values\nvocab=build_vocab(sentences)\nprint({k : vocab[k] for k in list(vocab)[:5]})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import operator\ndef check_coverage(vocab, embeddings_index):\n    oov={}\n    a={}\n    i,k=0,0\n    for word in tqdm(vocab):\n        try:\n            a[word]=embeddings_index[word]\n            k+= vocab[word]\n        except:\n            oov[word]=vocab[word]\n            i+=vocab[word]\n            pass\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab,embeddings1_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:20]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"For now we can see 2 main problems\n\n1.Contractions\n\n2.Punctuations\n\nBut what do we do with the punctuation then - Do we want to delete or consider as a token? It depends. If the token has an embedding, keep it, if it doesn't we don't need it anymore. So lets check:\n\nDouble-click (or enter) to edit","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"contraction_mapping = {\"ain't\": \"is not\", \"aren't\": \"are not\",\"can't\": \"cannot\", \"'cause\": \"because\", \"could've\": \"could have\", \"couldn't\": \"could not\", \"didn't\": \"did not\",  \"doesn't\": \"does not\", \"don't\": \"do not\", \"hadn't\": \"had not\", \"hasn't\": \"has not\", \"haven't\": \"have not\", \"he'd\": \"he would\",\"he'll\": \"he will\", \"he's\": \"he is\", \"how'd\": \"how did\", \"how'd'y\": \"how do you\", \"how'll\": \"how will\", \"how's\": \"how is\",  \"I'd\": \"I would\", \"I'd've\": \"I would have\", \"I'll\": \"I will\", \"I'll've\": \"I will have\",\"I'm\": \"I am\", \"I've\": \"I have\", \"i'd\": \"i would\", \"i'd've\": \"i would have\", \"i'll\": \"i will\",  \"i'll've\": \"i will have\",\"i'm\": \"i am\", \"i've\": \"i have\", \"isn't\": \"is not\", \"it'd\": \"it would\", \"it'd've\": \"it would have\", \"it'll\": \"it will\", \"it'll've\": \"it will have\",\"it's\": \"it is\", \"let's\": \"let us\", \"ma'am\": \"madam\", \"mayn't\": \"may not\", \"might've\": \"might have\",\"mightn't\": \"might not\",\"mightn't've\": \"might not have\", \"must've\": \"must have\", \"mustn't\": \"must not\", \"mustn't've\": \"must not have\", \"needn't\": \"need not\", \"needn't've\": \"need not have\",\"o'clock\": \"of the clock\", \"oughtn't\": \"ought not\", \"oughtn't've\": \"ought not have\", \"shan't\": \"shall not\", \"sha'n't\": \"shall not\", \"shan't've\": \"shall not have\", \"she'd\": \"she would\", \"she'd've\": \"she would have\", \"she'll\": \"she will\", \"she'll've\": \"she will have\", \"she's\": \"she is\", \"should've\": \"should have\", \"shouldn't\": \"should not\", \"shouldn't've\": \"should not have\", \"so've\": \"so have\",\"so's\": \"so as\", \"this's\": \"this is\",\"that'd\": \"that would\", \"that'd've\": \"that would have\", \"that's\": \"that is\", \"there'd\": \"there would\", \"there'd've\": \"there would have\", \"there's\": \"there is\", \"here's\": \"here is\",\"they'd\": \"they would\", \"they'd've\": \"they would have\", \"they'll\": \"they will\", \"they'll've\": \"they will have\", \"they're\": \"they are\", \"they've\": \"they have\", \"to've\": \"to have\", \"wasn't\": \"was not\", \"we'd\": \"we would\", \"we'd've\": \"we would have\", \"we'll\": \"we will\", \"we'll've\": \"we will have\", \"we're\": \"we are\", \"we've\": \"we have\", \"weren't\": \"were not\", \"what'll\": \"what will\", \"what'll've\": \"what will have\", \"what're\": \"what are\",  \"what's\": \"what is\", \"what've\": \"what have\", \"when's\": \"when is\", \"when've\": \"when have\", \"where'd\": \"where did\", \"where's\": \"where is\", \"where've\": \"where have\", \"who'll\": \"who will\", \"who'll've\": \"who will have\", \"who's\": \"who is\", \"who've\": \"who have\", \"why's\": \"why is\", \"why've\": \"why have\", \"will've\": \"will have\", \"won't\": \"will not\", \"won't've\": \"will not have\", \"would've\": \"would have\", \"wouldn't\": \"would not\", \"wouldn't've\": \"would not have\", \"y'all\": \"you all\", \"y'all'd\": \"you all would\",\"y'all'd've\": \"you all would have\",\"y'all're\": \"you all are\",\"y'all've\": \"you all have\",\"you'd\": \"you would\", \"you'd've\": \"you would have\", \"you'll\": \"you will\", \"you'll've\": \"you will have\", \"you're\": \"you are\", \"you've\": \"you have\" }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_contractions(text, mapping):\n    specials = [\"’\", \"‘\", \"´\", \"`\"]\n    for s in specials:\n        text = text.replace(s, \"'\")\n    text = ' '.join([mapping[t] if t in mapping else t for t in text.split(\" \")])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['question_text']=df_train['question_text'].progress_apply(lambda x: clean_contractions(x,contraction_mapping))\ndf_test['question_text']=df_test['question_text'].progress_apply(lambda x: clean_contractions(x,contraction_mapping))\n\nsentences= df_train['question_text'].apply(lambda x : x.split())\nvocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab,embeddings1_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"punct = \"/-'?!.,#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~\" + '\"\"“”’' + '∞θ÷α•à−β∅³π‘₹´°£€\\×™√²—–&'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def unknown_punct(embed, punct):\n    unknown = ''\n    for p in punct:\n        if p not in embed:\n            unknown += p\n            unknown += ' '\n    return unknown","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(unknown_punct(embeddings1_index, punct))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"punct_mapping = {\"‘\": \"'\", \"₹\": \"e\", \"´\": \"'\", \"°\": \"\", \"€\": \"e\", \"™\": \"tm\", \"√\": \" sqrt \", \"×\": \"x\", \"²\": \"2\", \"—\": \"-\", \"–\": \"-\", \"’\": \"'\", \"_\": \"-\", \"`\": \"'\", '“': '\"', '”': '\"', '“': '\"', \"£\": \"e\", '∞': 'infinity', 'θ': 'theta', '÷': '/', 'α': 'alpha', '•': '.', 'à': 'a', '−': '-', 'β': 'beta', '∅': '', '³': '3', 'π': 'pi', }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_special_chars(text, punct, mapping):\n    for p in mapping:\n        text = text.replace(p, mapping[p])\n    \n    for p in punct:\n        text = text.replace(p, f' {p} ')\n    \n    specials = {'\\u200b': ' ', '…': ' ... ', '\\ufeff': '', 'करना': '', 'है': ''}  # Other special characters that I have to deal with in last\n    for s in specials:\n        text = text.replace(s, specials[s])\n    \n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['question_text'] = df_train['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\ndf_test['question_text'] = df_test['question_text'].apply(lambda x: clean_special_chars(x, punct, punct_mapping))\n\nsentences= df_train['question_text'].apply(lambda x : x.split())\nvocab = build_vocab(sentences)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab,embeddings1_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov[:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling', 'counselling': 'counseling', 'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor', 'organisation': 'organization', 'wwii': 'world war 2', 'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary', 'Whta': 'What', 'narcisist': 'narcissist', 'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much', 'howmany': 'how many', 'whydo': 'why do', 'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does', 'mastrubation': 'masturbation', 'mastrubate': 'masturbate', \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum', 'narcissit': 'narcissist', 'bigdata': 'big data', '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend', 'airhostess': 'air hostess', \"whst\": 'what', 'watsapp': 'whatsapp', 'demonitisation': 'demonetization', 'demonitization': 'demonetization', 'demonetisation': 'demonetization', 'pokémon': 'pokemon'}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def correct_spelling(x, dic):\n    for word in dic.keys():\n        x = x.replace(word, dic[word])\n    return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df_train['question_text'] = df_train['question_text'].progress_apply(lambda x: correct_spelling(x, mispell_dict))\ndf_test['question_text'] = df_test['question_text'].apply(lambda x: correct_spelling(x, mispell_dict))\n\nsentences= df_train['question_text'].progress_apply(lambda x : x.split())\nsentences = [[word for word in sentence] for sentence in tqdm(sentences)]\nvocab = build_vocab(sentences)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"oov = check_coverage(vocab,embeddings1_index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#del sentences,build_vocab,vocab,oov,mispell_dict,punct,contraction_mapping\n#del check_coverage","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len_voc = 95000\nmax_len =60","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t = Tokenizer(num_words=len_voc, filters='')\nt.fit_on_texts(df_train['question_text'])\nX = t.texts_to_sequences(df_train['question_text'])\nX_test=t.texts_to_sequences(df_test['question_text'])\nX = pad_sequences(X, maxlen=max_len)\nX_test = pad_sequences(X_test, maxlen=max_len)\nword_index=t.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y = df_train['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=0.1, random_state=420)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del df_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"   def make_embed_matrix(embeddings_index, word_index, len_voc):\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    word_index = word_index\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (len_voc, embed_size))\n    \n    for word, i in word_index.items():\n        if i >= len_voc:\n            continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: \n            embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding = make_embed_matrix(embeddings1_index, word_index, len_voc)\n\ndel word_index\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"early = EarlyStopping(monitor='val_loss', mode=\"min\", patience=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lstm = Sequential()\nlstm.add(Embedding(len_voc, 300, weights=[embedding], trainable=False))\nlstm.add(Bidirectional(LSTM(units = 256, return_sequences= True)))\nlstm.add(Dropout(rate = 0.2))\n#lstm.add(Bidirectional(LSTM(units = 60)))\n#lstm.add(Dropout(rate = 0.2))\nlstm.add(Conv1D(128, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\"))\n\nlstm.add(GlobalMaxPooling1D())\n#lstm.add(Dense(units = 64, activation = 'relu'))\nlstm.add(Dropout(rate = 0.2))\nlstm.add(Dense(units = 1, activation = 'sigmoid'))\n\nlstm.compile(optimizer = 'adam', loss = 'binary_crossentropy', metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lstm.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs = 3\nbatch_size = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"hist = lstm.fit(X_train, y_train, batch_size=batch_size, epochs=epochs, validation_data=(X_val,y_val),callbacks=[early])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(12,8))\nplt.plot(hist.history['loss'], label='Train Loss')\nplt.plot(hist.history['val_loss'], label='Test Loss')\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import confusion_matrix,classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val_y = lstm.predict([X_val], batch_size=1024, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_val_y = (pred_val_y > 0.50).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(y_val,pred_val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(confusion_matrix(y_val,pred_val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test=lstm.predict([X_test], batch_size=1024, verbose=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred_test = (pred_test > 0.50).astype(int)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"out_df = pd.DataFrame({\"qid\":df_test[\"qid\"].values})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"out_df['prediction'] = pred_test","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"out_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}