{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"This code is a complementary for the main code: https://www.kaggle.com/eligijus/rnn-spell-checker \n\nThis code is used to search for the optimal treshold when to label a question as sincere or insincere. This process takes a lot of time and the main kernel sometimes exceeds the time limit."},{"metadata":{"_uuid":"218007787ee9136540167806852345b1e4073d06"},"cell_type":"markdown","source":"## Input"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom sklearn.metrics import f1_score\nfrom nltk.util import ngrams\nimport pandas as pd\nimport string\nimport re\nfrom gensim.models import KeyedVectors\nimport numpy as np\nfrom keras.models import Sequential, Input\nfrom keras.layers import Dense, Dropout, LSTM, Embedding, GlobalMaxPool1D, Conv1D, MaxPooling1D \nfrom keras.layers import CuDNNLSTM, Bidirectional, CuDNNGRU, GlobalAvgPool1D, concatenate\nfrom keras.models import Model\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nimport tensorflow as tf\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.optimizers import Optimizer\nfrom keras.callbacks import *\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nimport gc\nimport Levenshtein \nfrom tqdm import tqdm\nimport operator\n\n## Reading input\n\nd = pd.read_csv('../input/train.csv')\nY_train = d['target']\nX_train = d['question_text'] \n\n## Reading the test set \n\nd_test = pd.read_csv('../input/test.csv')\nX_test = d_test['question_text']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"af7c9acb7c244a88c40ef947a7998ef2b54b4bcf"},"cell_type":"markdown","source":"# Custom functions"},{"metadata":{"trusted":true,"_uuid":"3f0f8fcc19704cd3356013cb2ef6419ab6b9f467"},"cell_type":"code","source":"## Preprocesing functions\n\ndef spell_checker(string_vec, vocab_mapper):\n    \"\"\"\n    A function to change the word in a dictionary \"x\" : \"y\"\n    in the following manner: x -> y\n    \"\"\"\n    cleaned_strings = []\n    for char in string_vec:\n        cleaned_words = []\n        for x in char.split():\n            if vocab_mapper.get(x) is not None:\n                x = vocab_mapper.get(x)\n            cleaned_words.append(x)\n        cleaned_words = ' '.join(cleaned_words)\n        cleaned_strings.append(cleaned_words)\n    \n    return pd.Series(cleaned_strings) \n\ndef encode_digit(x):\n    \"\"\"\n    Encodes a digit in a string\n    \"\"\"\n    x = re.sub('[0-9]{5,}', ' ##### ', x)\n    x = re.sub('[0-9]{4}', ' #### ', x)\n    x = re.sub('[0-9]{3}', ' ### ', x)\n    x = re.sub('[0-9]{2}', ' ## ', x)\n    return x\n\ndef clean_digits(string_vec):\n    \"\"\"\n    Removes digits from a string vector\n    \"\"\"\n    cleaned_string = [encode_digit(s) for s in string_vec]\n    \n    return pd.Series(cleaned_string)\n\ndef clean_ws(string_vec):\n    \"\"\"\n    Cleans whitespaces\n    \"\"\"\n    cleaned_string = [re.sub( '\\s+', ' ', s).strip() for s in string_vec]\n    return pd.Series(cleaned_string)\n\ndef clean_word(char, punct):\n    \"\"\"\n    A function that removes bad punctuations and splits good ones in a given string\n    \"\"\"\n    for p in punct:\n        char = char.replace(p, f' {p} ')\n    \n    return(char)\n\ndef clean_punct(string_vec, punct):\n    \"\"\"\n    Function that cleans the punctuations\n    \"\"\"\n    cleaned_string = []\n    for char in tqdm(string_vec):\n        char = [clean_word(x, punct) for x in char.split()]\n        cleaned_string.append(' '.join(char))\n    return pd.Series(cleaned_string)   \n\ndef tokenize_text(string_vec, tokenizer, max_len):\n    \"\"\"\n    Tokenizes a given string vector\n    \"\"\"\n    token = tokenizer.texts_to_sequences(string_vec)\n    token = pad_sequences(token, maxlen = max_len)\n    \n    return token\n\n## Functions for embeddings\n\ndef load_from_text(path):\n    \"\"\"\n    A functions that load embeddings from a txt document\n    \"\"\"\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(path, errors='ignore'))\n    return embeddings_index\n\ndef read_embedding(path, reading_type, binary = False):\n    \"\"\"\n    Reads the embeddings from a .txt or .vec file\n    \"\"\"\n    if(reading_type == 'text'):\n        model_embed = load_from_text(path)\n        \n    if(reading_type == 'word2vec'):\n        model_embed = KeyedVectors.load_word2vec_format(path, binary = binary)\n    \n    return model_embed   \n\ndef create_embedding_matrix(model_embed, tokenizer, max_features, embed_size):\n    \"\"\"\n    Creates an embeding matrix\n    \"\"\"\n    embedding_matrix = np.zeros((max_features, embed_size))\n    for word, index in tokenizer.word_index.items():\n        if index > max_features - 1:\n            break\n        else:\n            try:\n                embedding_matrix[index] = model_embed[word]\n            except:\n                continue\n    return embedding_matrix        \n\ndef tokenize_text(string_vec, tokenizer, max_len):\n    \"\"\"\n    Tokenizes a given string vector\n    \"\"\"\n    token = tokenizer.texts_to_sequences(string_vec)\n    token = pad_sequences(token, maxlen = max_len)\n    \n    return token\n    \n### Functions for predictions\n\ndef to_binary(p_array, treshold):\n    \"\"\"\n    Converts the prediction from probability to 0 or 1\n    \"\"\"\n    y_hat = []\n    for i in range(len(p_array)):\n        if p_array[i] > treshold:\n            y_hat.append(1)\n        else:\n            y_hat.append(0)\n    return y_hat    \n\ndef optimal_treshold(y, yhat):\n    \"\"\"\n    Computes the otpimal treshold for the f1 statistic\n    \"\"\"\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i/100 for i in range(10,90)]:\n        score = f1_score(y, yhat > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result\n\n## Feature engineering\n\ndef share_unique(string_vec):\n    \"\"\"\n    A function that calculates the share of unique words in a given string\n    \"\"\"\n    share_list = []\n    for char in string_vec:\n        sh = len(set(char.split()))/len(char.split())\n        share_list.append(sh)\n    return share_list\n    \ndef share_capslock(string_vec):\n    \"\"\"\n    Calculates the share of caps locked words in a given string\n    \"\"\"\n    share_list = []\n    for char in string_vec:\n        to_upper = char.upper().split()\n        sh = len(set(char.split()).intersection(to_upper))/len(char.split())\n        share_list.append(sh)\n    return share_list    \n    \n## Functions for deep learning\n    \ndef f1(y_true, y_pred):\n    '''\n    metric from here \n    https://stackoverflow.com/questions/43547402/how-to-calculate-f1-macro-in-keras\n    '''\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"121aa06e6cd8196a52c5718706fbe6c79695651c"},"cell_type":"markdown","source":"# Preprocesing pipeline"},{"metadata":{"trusted":true,"_uuid":"be988ffabf12727496c971e640fa39c958e67949"},"cell_type":"code","source":"## Word fixing:\n\nvocab_mapper = {'Quorans' : 'Qurans', \n                'Blockchain' : 'blockchain', \n                'blockchains' : 'blockchain',\n                'demonetisation' : 'demonetization', \n                'ethereum' : 'Ethereum', \n                'Qoura' : 'Quora', \n                'SJWs' : 'SJW', \n                'bhakts' : 'bhakti', \n                'Bhakts' : 'Bhakti', \n                'kotlin' : 'Kotlin', \n                'narcissit' : 'narcissist', \n                'Trumpism' : 'Trump', \n                'Tamilans' : 'Tamilians', \n                'acturial' : 'actuarial', \n                'demonitization' : 'demonetization', \n                'Demonetization' : 'demonetization',\n                'Whyis' : 'Why is', \n                'AirPods' : 'AirPod', \n                'Drumpf': 'Trumpf', \n                'Zhihu' : 'Zhihua', \n                'Neuralink' : 'Neurolink', \n                'fullform' : 'full-form', \n                'biharis' : 'Biharis', \n                'madheshi' : 'Madheshi', \n                'Xiomi' : 'Xiaomi', \n                'rohingya' : 'Rohingya', \n                'Despacito' : 'Desposito', \n                'schizoids' : 'schizoid', \n                'MHTCET' : 'MHT-CET', \n                'fortnite' : 'Fortnite',\n                'Bittrex' : 'Bitrex', \n                'ReactJS' : 'JavaScript', \n                'hyperloop' : 'Hyperloop', \n                'adhaar' : 'Aadhaar', \n                'Adhaar' : 'Aadhaar', \n                'Baahubali' : 'Bahubali', \n                'Cryptocurrency' : 'cryptocurrency', \n                'cryptocurrencies' : 'cryptocurrency',\n                '\\u200b':\" \", \n                \"\\ufeff\" : \"\",\n                'mastuburation' : 'masturbation',\n                'quara' : 'Quora',\n                'Quoras' : 'Quora',\n                \"fiancé\" : \"fiance\", \n                'π' : 'pi', \n                'Pokémon' : 'Pokemon' }\n\npunct = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']\npunct = ''.join(punct)\n\n## Defining the preprocesing\n\ndef preproc_pipeline(string_df, model_embed, punct, word_expansion = None, vocab_mapper = None):\n    \"\"\"\n    The whole pipeline of cleaning\n    \"\"\"\n    \n    if(word_expansion is not None):\n        string_df = spell_checker(string_df, word_expansion)\n        \n    string_df = clean_punct(string_df, punct)\n    \n    if(vocab_mapper is not None):\n        string_df = spell_checker(string_df, vocab_mapper)\n    \n    string_df = clean_digits(string_df)\n    string_df = clean_ws(string_df)\n    return string_df","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b8524e04151ad76af831291e986598b3d8304e8"},"cell_type":"markdown","source":"# Reading the embeddings"},{"metadata":{"trusted":true,"_uuid":"08ce4add04ef16a94d2d0f034f9af30d7a1d1cb3"},"cell_type":"code","source":"embed_dict = {'google': {'path' : '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',\n                         'reading' : 'word2vec', \n                         'binary': True}, \n                         \n               'glove': {'path': '../input/embeddings/glove.840B.300d/glove.840B.300d.txt', \n                         'reading' : 'text',\n                         'binary': False}, \n               \n                'wiki': {'path' : '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec', \n                         'reading' : 'text', \n                         'binary': False}, \n                         \n               'paragram': {'path': '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',\n                         'reading' : 'text', \n                          'binary' : False}\n}\n\nembeds_to_use = ['glove'] ## name of key in the 'embed_dict' dictionary\n\nindex = 0\nfor embed in embeds_to_use:\n    \n    print('Reading: ' + embed)\n    path = embed_dict.get(embed)['path']\n    type_of_file = embed_dict.get(embed)['reading']\n    bin = embed_dict.get(embed)['binary']\n    embedding = read_embedding(path, type_of_file, bin)\n\n    if embed == embeds_to_use[0]:\n        model_embed = embedding\n    else: \n        all_keys = [k for k in embedding.keys()]\n        index += 1\n        for key in tqdm(all_keys):\n            try:\n                vect = np.concatenate([model_embed[key], embedding[key]])\n            except:\n                vect = np.concatenate([np.zeros((1, 300 * index), dtype = 'float32')[0], embedding[key]])\n                \n            model_embed.update({key : vect})    \n            del vect\n        del all_keys    \n            \n    del embedding\n    gc.collect()\n    time.sleep(5)    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"61c915d8714eafcb2fa25ef8ef58d1c953469826"},"cell_type":"markdown","source":"# Model and hyperparameters"},{"metadata":{"trusted":true,"_uuid":"5ee1d1b7273812c1a091feec370a6c7ead157077"},"cell_type":"code","source":"## Main model \n\ndef RNN_model(maxlen, embed_size, max_features, embedding_matrix,\n              loss_f = 'binary_crossentropy', opti = 'adam', metr = f1):\n    \n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n    x = Bidirectional(CuDNNLSTM(128, return_sequences = True))(x)\n    x = Bidirectional(CuDNNLSTM(100, return_sequences = True))(x)\n    x = Bidirectional(CuDNNLSTM(72, return_sequences = True))(x)\n    \n    atten = Attention(maxlen)(x)\n    avg_pool = GlobalAvgPool1D()(x)\n    max_pool = GlobalMaxPool1D()(x)\n    \n    conc = concatenate([atten, avg_pool, max_pool])\n    conc = Dense(64, activation=\"relu\")(conc)\n    outp = Dense(1, activation=\"sigmoid\")(conc)   \n\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss = loss_f, optimizer = opti , metrics=[metr])\n    return model\n\n## Hyperparameters\n\nmaxlen = 70 ## Number of words in each sentence to use\nmax_features = 120000 \n\nbatch_size = 512 ## batch size\nnumb_epoch = 2 ## number of epochs\n\n## Size of the embedding\n### If we used more than one embedding, then the number of coordinates for each word is doubled, tripled, etc.\n\nembed_size = 300 * len(embeds_to_use)\nuse_k_fold = False","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f8052a6ffef1dbb279063b421dee6cc4cff7fbd8"},"cell_type":"markdown","source":"# Applying the preprocesing"},{"metadata":{"trusted":true,"_uuid":"1f120115b4046d136c19d39b47c23c7f53901338"},"cell_type":"code","source":"X_tr = preproc_pipeline(X_train, model_embed, punct, vocab_mapper = vocab_mapper)\nX_te = preproc_pipeline(X_test, model_embed, punct, vocab_mapper = vocab_mapper)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4b485158d2c7aafedf049adb2ba9156068a25078"},"cell_type":"markdown","source":"# K fold analysis"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"dff830a25adc49eb2ff38d53ae078ab3d10d8707"},"cell_type":"code","source":"## Reshuffling all the data to insure the same distribution in all folds\n\nX_tr = X_tr.iloc[np.random.permutation(len(X_tr))]\nY_tr = Y_train.iloc[X_tr.index.values]\n\n## Reseting the indexes\n\nX_tr = X_tr.reset_index(drop = True)\nY_tr = Y_tr.reset_index(drop = True)\n\nk_fold = 5\nt_results = [] ## Object to store the optimal tresholds in\n\n## Looping through all the folds\n\nk_fold_index = np.array_split(X_tr.index.values, k_fold)\nfor k in range(k_fold):\n\n    print('Fold: ' + str(k + 1))\n\n    ## Creating the training and test sets\n\n    test_index = k_fold_index[k]\n    not_k = [x for x in set(range(k_fold)) - set([k])]\n    train_index = np.concatenate([k_fold_index[x] for x in not_k])\n\n    XX_tr = X_tr.iloc[train_index]\n    YY_tr = Y_tr.iloc[train_index] \n\n    XX_te = X_tr.iloc[test_index]\n    YY_te = Y_tr.iloc[test_index]\n\n    ## Text tokenization\n\n    tokenizer = Tokenizer(num_words = max_features)\n    tokenizer.fit_on_texts(list(XX_tr))\n\n    XX_tr = tokenize_text(XX_tr, tokenizer, maxlen)\n    XX_te = tokenize_text(XX_te, tokenizer, maxlen)   \n\n    ## Creating a weight matrix for words in training matrix\n\n    embedding_matrix = create_embedding_matrix(model_embed, \n                                                tokenizer, \n                                                max_features, \n                                                embed_size)\n\n    ## Creating the model \n\n    model = RNN_model(XX_tr.shape[1], embed_size, max_features, embedding_matrix)\n\n    model_fited = model.fit(\n    XX_tr,\n    YY_tr.values, \n    batch_size = batch_size, \n    nb_epoch = numb_epoch)  \n\n    ## Predictions\n\n    y_hat_probs = model.predict(XX_te)\n\n    ## Optimal treshold\n\n    opti_t = optimal_treshold(YY_te.values, y_hat_probs)\n    print(opti_t)\n    t_results.append(opti_t['threshold'])\n\n    ### Releasing memory\n\n    del embedding_matrix, model, XX_tr, XX_te, tokenizer, model_fited\n    gc.collect()\n    time.sleep(5)\n\n## Averaging the optimal tresholds\n\nopti_t = np.mean(t_results)\nprint(\"Optimal treshold: \" + str(opti_t))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}