{"cells":[{"metadata":{"_uuid":"27a371602dcb7d06a7e0a0ea8b60ba71e99d49fc"},"cell_type":"markdown","source":"Resources \n\nI have used the codes from https://www.kaggle.com/shujian/blend-of-lstm-and-cnn-with-4-embeddings-1200d       and adapted for the tokenizer preprocessing, the loading of the Google word embeddings and the construction of the embedding matrix"},{"metadata":{"_uuid":"5932e9afee6d7e6686338e12fd6fa7dd4d8e1a84"},"cell_type":"markdown","source":"# Introduction"},{"metadata":{"_uuid":"7c7f8510cdc2cbd4e15eb1ace2ffc2c5ff67c577"},"cell_type":"markdown","source":"# 1. Load packages and data"},{"metadata":{"trusted":true,"_uuid":"625c9a359ce617b071371a6593e64ac92af4592e"},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport nltk\nimport re\nimport time\nimport random\nimport matplotlib.pyplot as plt\nimport sklearn\nfrom numpy import genfromtxt\nfrom tqdm import tqdm\n\n\n#nltk.download('book')\nfrom keras.preprocessing.text import Tokenizer\nfrom nltk.corpus import stopwords\nfrom nltk.corpus import wordnet\nfrom nltk.book import *\n#from gensim.models import Word2Vec\nfrom string import punctuation\nfrom nltk.tokenize import word_tokenize","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ca84900f96ecd3b8bf0fc3566dc1be96083c4a7b"},"cell_type":"markdown","source":""},{"metadata":{"trusted":true,"_uuid":"9284d024fa10f1fd98a03e144060049097f5263b"},"cell_type":"code","source":"train_data = pd.read_csv(\"../input/train.csv\")\ndel train_data['qid']\n\n\ntest_data = pd.read_csv(\"../input/test.csv\")\n\nshare = sum(train_data['target'] == 0) / len(train_data['target'])\n\nprint(\"The share of non insult comments is\", round(share,4) * 100, \"%\")\ntrain_data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d282f3ff7999f5977d38f61889da765afd90c8d9"},"cell_type":"markdown","source":"# 2. Data preprocessing"},{"metadata":{"_uuid":"c64ba8ef83483a7cb472115790e59ef974cd1bb8"},"cell_type":"markdown","source":"In this section several functions are defined to clean and preprocess the dataset for further use.\n\nIn order to be consistent with writing certain phrases we will implement the function $decontracted()$, which will convert phrases like \"don't\" to \"do not\" etc. "},{"metadata":{"trusted":true,"_uuid":"60600566705b2f64c23c7d7fd58bb24b6ea4f8cb"},"cell_type":"code","source":"def decontracted(phrase):\n    \n    \"\"\"\n    function that takes as input the most used english phrases and expands them to the actual\n    words\n    \n    Input: \n    phrase - Phrases like \"won't\" or \"don't\"\n\n    Returns: \n    The same phrase expanded to \"will not\" and \"do not\" respectively\n    \"\"\"\n    \n    phrase = re.sub(r\"won't\", \"will not\", phrase)\n    phrase = re.sub(r\"won’t\", \"will not\", phrase)\n    phrase = re.sub(r\"dont\", \"do not\", phrase)\n    hrase = re.sub(r\"don’t\", \"do not\", phrase)\n    phrase = re.sub(r\"don't\", \"do not\", phrase)\n    phrase = re.sub(r\"can\\'t\", \"cannot\", phrase)\n    phrase = re.sub(r\"can't\", \"cannot\", phrase)\n    phrase = re.sub(r\"can’t\", \"cannot\", phrase)\n    phrase = re.sub(r\"n\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"n't\", \" not\", phrase)\n    phrase = re.sub(r\"n’t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'re\", \" are\", phrase)\n    phrase = re.sub(r\"'re\", \" are\", phrase)\n    phrase = re.sub(r\"’re\", \"are\", phrase)\n    phrase = re.sub(r\"\\'s\", \" is\", phrase)\n    phrase = re.sub(r\"’s\", \"is\", phrase)\n    phrase = re.sub(r\"'s\", \"is\", phrase)\n    phrase = re.sub(r\"\\'d\", \" would\", phrase)\n    phrase = re.sub(r\"'d\", \" would\", phrase)\n    phrase = re.sub(r\"’d\", \" would\", phrase)\n    phrase = re.sub(r\"\\'ll\", \" will\", phrase)\n    phrase = re.sub(r\"'ll\", \" will\", phrase)\n    phrase = re.sub(r\"’ll\", \" will\", phrase)\n    phrase = re.sub(r\"\\'t\", \" not\", phrase)\n    phrase = re.sub(r\"'t\", \" not\", phrase)\n    phrase = re.sub(r\"’t\", \" not\", phrase)\n    phrase = re.sub(r\"\\'ve\", \" have\", phrase)\n    phrase = re.sub(r\"'ve\", \" have\", phrase)\n    phrase = re.sub(r\"’ve\", \" have\", phrase)\n    phrase = re.sub(r\"\\'m\", \" am\", phrase)\n    phrase = re.sub(r\"'m\", \" am\", phrase)\n    phrase = re.sub(r\"’m\", \" am\", phrase)\n    phrase = re.sub(r'\\w*@\\w*','', phrase)\n    \n    return phrase\n\n\ndef preproc(data,word):\n\n\n    sen = []\n\n    for i in range(len(data)):\n        sen.append(re.split(' |\\\\\\\\n|\\\\\\\\|\\n\\n|\\n|xc2|xa0|x80|xe2|!|\"|\\.(?!\\d)|\\?(?!\\d)|-|,',data[word][i]))\n\n    for i in range(len(data)):  \n        sen[i] = [word.lower() for word in sen[i]]\n        sen[i] = [decontracted(word) for word in sen[i]]\n    \n    punct = list(punctuation)\n    punct.append('``')\n    punct.append(\"''\")\n    punct.append('--')\n    punct.append('...')\n    punct.append('')\n    punct.append(',')\n    punct.append(\"'\")\n\n    sentences = []\n    \n    for i in range(len(sen)):\n        sentences.append([word for word in sen[i] if word not in punct])\n\n        \n    data = [' '.join(i) for i in sentences]\n    data = np.asarray(data)    \n    \n    \n    #[data[i].split() for i in range(len(data))]\n        \n    return data  \n\n\n\ndef word_length(data):\n\n    length = []\n\n    for i in range(len(data)):\n        length.append(len(data[i].split()))\n    \n    max_len = max(length)\n    return max_len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a91485ae75dac5ecb29efd74236852b014e25b8e"},"cell_type":"code","source":"phrase = decontracted(\"I don't like this movie, I won't watch it again, I wasn’t in the house\")\nprint('The sentence is going to be:',phrase)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c320e7b2a08a9d7108755bbe35b13e5a263b02c7"},"cell_type":"markdown","source":"The next function is used to make the data useable. This means that words are going to be split at stoping signs and all words are going to be written in lower case etc. Also the function $decontracted$ will be used in this function called $preproc$. At this point we are just preprocessing the data and are not investigating whether the sentences make sense, this will be adressed in the next section."},{"metadata":{"trusted":true,"_uuid":"02d7cc82c25fa617886d137d9256a7760cda06cb"},"cell_type":"code","source":"data = preproc(train_data,'question_text')\ndata_test = preproc(test_data,'question_text')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"eab3cc333ac063926f6871923a48269adc99c2d2"},"cell_type":"code","source":"all_data = np.concatenate((data,data_test),axis = 0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"267c713d1922cbc71869e3cd3ec2097071538aab"},"cell_type":"code","source":"tokenizer = Tokenizer()\ntokenizer.fit_on_texts(all_data)\ntokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9aa643a49420f5f87960b0aefaee22aa52b2cd10"},"cell_type":"code","source":"target = train_data['target']#[index]\ntarget = np.asarray(target)\n  \n\na = np.zeros(len(data))\n\nfor i in range(0,len(data)):\n    a[i] = len(data[i].split())\n\nprint(sum(a<=90) / len(data))    \n    \ndata = data[a <= 90]\ntarget = target[a <=90]\n\n#####################################\n\n\ndf = pd.DataFrame({'Comment': data, 'y':target})\n\nX = df['Comment']\ny = df['y']\n\n\nmax_len2 = 90\nprint(max_len2)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9bf11133eafce896ac7d38846f085ff1df717b14"},"cell_type":"code","source":"df_test = pd.DataFrame({'Comment': data_test})\n\ntest_X = df_test['Comment']\n\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dec6fccc71fe110731b0e0f648dcb2a5eb1e7b53"},"cell_type":"code","source":"del data, data_test, all_data","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f676e0f725daa576316f7f7079a53593275de19b"},"cell_type":"markdown","source":"# 3. Word Embeddings & sentence indices "},{"metadata":{"_uuid":"13690382fd9e2316c80b99b5bd8405b49c8eeeed"},"cell_type":"markdown","source":"Here we are going to define the dictionary of the word embedding to be used as input for the model later and the positions of the words in the dictionary as input to the model."},{"metadata":{"trusted":true,"_uuid":"41e1a06f204a872c082b10078969a6173e923002"},"cell_type":"code","source":"words_to_vec = {}\nf = open('../input/embeddings/glove.840B.300d/glove.840B.300d.txt')\nfor line in tqdm(f):\n    values = line.split(\" \")\n    word = values[0]\n    coefs = np.asarray(values[1:], dtype='float32')\n    words_to_vec[word] = coefs\nf.close()\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a2c0a7b01fe03d3f731a21f11059017d5650d78a"},"cell_type":"code","source":"all_embedds = np.stack(words_to_vec.values())\nembedd_mean = all_embedds.mean()\nembedd_std = all_embedds.std()\n\nword_index = tokenizer.word_index\ncol_shape = words_to_vec['one'].shape[0]\n\nembedd_matrix = np.random.normal(embedd_mean, embedd_std, (len(word_index),col_shape))\n\nfor word, i in word_index.items():\n    if words_to_vec.get(word) is not None:\n        embedd_matrix[i-1,:] = words_to_vec[word]\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9c625f703332e17bab85db9969797a0f418d611e"},"cell_type":"code","source":"i = 0\nword_to_index = {}\n\n\nfor word in word_index.keys():\n    word_to_index[word] = i\n    i = i + 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5401f6bdb7f7a911fd54b11160e31030dfb56e5"},"cell_type":"code","source":"#words_to_vec['is']\n#word_to_index['is']\nsum(embedd_matrix[2,:] - words_to_vec['what'])\n\ndel words_to_vec","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"af5d4df851c1eb33a413f6604f5b609cc45cfd67"},"cell_type":"code","source":"def sentence_to_index(data,word_to_index,max_len,temp):\n    \n    \"\"\"\n    function that takes a sentences and gives the vector of indices back for all the words in the sentence\n    \n    Input: \n    data ... That is the data set, which contains the sentences to be translated to indices \n    word_to_index ... The dictionary that holds the index of any word in the word embedding \n    max_len... Maximum length of a sentence. If a sentence does not have maximum length, then the additional fields \n                are filled with zeros \n    temp ... This function is used twice\n                1. Identify how many sentences have words which are not in the Glove6B embedding. If there are too\n                   many unidentifable words, then this sentences will be taken out of the data (temp == 0)\n                2. For the actual indexing of the vectors to find out the word vector of certain words.\n                \n    Output:\n    Index_vector ... A matrix which returns the index of every word of a sentence in the corresponding word embedding\n    \"\"\"\n\n    m = data.shape[0] # number of traing examples\n    index_vector = np.zeros((m,max_len),dtype = 'int32') # Matrix of all sentence examples and corresponding indices\n    \n    for i in range(m):\n        # Standardize all words in the sentence to lower case and split them \n        sentence_words = data[i].lower().split()\n        \n        j = 0\n        \n        for word in sentence_words:\n            if word in word_to_index.keys():\n                index_vector[i,j] = word_to_index[word]\n            elif temp == 0:\n                index_vector[i,j] = -1\n            else:\n                index_vector[i,j] = 0\n            j = j + 1\n              \n    return index_vector","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"026f38877af4b698c97bc45931d02aa3abb66bfd"},"cell_type":"markdown","source":"If there are to many words in a comment which are not part of the dictionary these comments will be taken out of the dataset. The reason is that the model is learning on reliable and meaningful data."},{"metadata":{"_uuid":"28c5e7cc680dcad49696940f0786a60e55549f31"},"cell_type":"markdown","source":"# 4. Models "},{"metadata":{"trusted":true,"_uuid":"27b70ca1de8d817c4ed2fcf1fb8629e77eec4ebc"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.02)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef40bdcd9c77f2bc6719215c303b69fc35ccb8a7"},"cell_type":"markdown","source":"## 4.2. RNN Model"},{"metadata":{"trusted":true,"_uuid":"7fd1ea52c8a771d1b0a58bef818c802dabdf8e68"},"cell_type":"code","source":"from keras.models import Model,Sequential\nfrom keras.layers import Dense, Input, Dropout, LSTM, Activation, LeakyReLU,GRU,Flatten,MaxPooling1D,Bidirectional,GlobalMaxPooling1D,Conv1D,Conv2D, MaxPooling2D\nfrom keras.layers.embeddings import Embedding\nfrom keras.preprocessing import sequence\nfrom keras.optimizers import Adam\nfrom keras.models import load_model\nfrom sklearn.utils import class_weight\nfrom keras.callbacks import EarlyStopping,ModelCheckpoint\nfrom keras import regularizers","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"03e6c4de9781984f85038dee3f8c3f97ff81166d"},"cell_type":"markdown","source":"Here we are adding the embedding layer that is going to use the first layer after the input layer to transform the index vector to their corresponding word vectors. "},{"metadata":{"trusted":true,"_uuid":"e33032f6f3e142f539519d1b0fe1336052535d20"},"cell_type":"code","source":"vocab_len = len(word_to_index) \nembedding_dim = 300\n    \nembedding_layer = Embedding(input_dim = vocab_len, output_dim = embedding_dim, weights = [embedd_matrix], trainable = False)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"45807352730973550a854c55bac246f491245cba"},"cell_type":"code","source":"#input_shape = (max_len2,)\n\n\nsentence_indices = Input(shape = (90,), dtype = 'int32')\n    \n\n# Propagate sentence_indices through your embedding layer, you get back the embeddings\nembeddings = embedding_layer(sentence_indices)\nX = Dropout(0.4)(embeddings)\n# Propagate the embeddings through an LSTM layer with 128-dimensional hidden state\n# Be careful, the returned output should be a batch of sequences.\nX = Bidirectional(GRU(units = 64, activation = 'tanh',return_sequences = True))(X)\n# Add dropout with a probability of 0.7\nX = Dropout(0.4)(X)\nX = GlobalMaxPooling1D()(X)\n\nX = Dense(units = 1)(X)\n# Add a softmax activation\nX = Activation('sigmoid')(X)\n    \n# Create Model instance which converts sentence_indices into X.\nmodel = Model(inputs = sentence_indices, outputs = X)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"29be8a008fb3404139494abfe8c5d9d391e9801d"},"cell_type":"markdown","source":"Model specifications that are going to be used to optimize the model and find the best solution for the validation set"},{"metadata":{"trusted":true,"_uuid":"8b1d29ee1e2d1453c51f94efd894606607aae33f"},"cell_type":"code","source":"X_train_index = sentence_to_index(np.asarray(X_train),word_to_index,max_len2,temp = 1)\nY_train_index = np.asarray(y_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9fb61b5e963d2bb6be12c6776eb26c2c3f600519"},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e84d4f67aae6e237af98a9718ff9bb8834d7dae0"},"cell_type":"code","source":"opt = Adam(lr=0.001,decay = 10e-6)\nmodel.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6f74dd748ef3b54effec810e530571671d6eaa03"},"cell_type":"code","source":"class_weight = {0: 1.,\n                1: 1.}","execution_count":null,"outputs":[]},{"metadata":{"scrolled":false,"trusted":true,"_uuid":"bd4cc3262d1f5b9210aabe2397863c54a3b87c49"},"cell_type":"code","source":"checkpoint = ModelCheckpoint('model1_check.h5', monitor='val_acc', verbose=1, save_best_only=True, mode='max')\ntemp1 = model.fit(X_train_index, Y_train_index, validation_split = 0.01,epochs = 2, batch_size = 1024,class_weight = class_weight, callbacks=[checkpoint])\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"81a75735a54558307607ac45b4ac297bb672007d"},"cell_type":"markdown","source":"## 4.3. Model evaluation"},{"metadata":{"trusted":true,"_uuid":"dea639637363abf7424286a5127b9b5145f1019d"},"cell_type":"code","source":"X_test_index = sentence_to_index(np.asarray(X_test), word_to_index, max_len = max_len2, temp = 1)\n#Y_test_index = np.eye(2)[np.asarray(y_test).reshape(-1)]\nY_test_index = np.asarray(y_test)\nloss, acc = model.evaluate(X_test_index, Y_test_index)\nprint()\nprint(\"Test accuracy = \",acc * 100, \"%\")\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1735ac5d2bed69676817ac8602b0e67ed1ed5c71"},"cell_type":"markdown","source":"Now we are going to predict and calculate the F1 score and predict comments we have not seen before. There we are going to use the sklearn function metrics.F1_score."},{"metadata":{"trusted":true,"_uuid":"9d64ad18d0e49985a6078a5fa2dc0a242c7519f6"},"cell_type":"code","source":"y_pred = model.predict(sentence_to_index(np.asarray(X_test), word_to_index, max_len2, temp = 1))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"efee87c67615263d3c2e2d364da217be483f7874"},"cell_type":"code","source":"#Finding the best value for F1 score threshold\nF1score = {}\n\nfor n in np.arange(0.0, 0.51, 0.01):\n    F1score[n] = sklearn.metrics.f1_score(y_test,y_pred > n)\n\n\n\nimport operator\nthreshold = max(F1score.items(), key=operator.itemgetter(1))[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe4d0dd332c2bb62881a65ae8a4a98f8cc2f82ba"},"cell_type":"code","source":"test_predict = model.predict(sentence_to_index(np.asarray(test_X), word_to_index, max_len2, temp = 1))\nprediction =  test_predict > threshold\npred = prediction * 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3f4f2457d35c9759d51e18e62594adbdfed80a62"},"cell_type":"code","source":"test_data['prediction'] = pred\ndel test_data['question_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe6b4eb130604f8f8813894a21b30b3b9d82f806"},"cell_type":"code","source":"test_data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"383f74175dcc4882574e9fc01fb034cfeb8214ea"},"cell_type":"code","source":"test_data.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1aa0c6f0ccc6a788322e9c8b0d0bf118a1ec7440"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}