{"cells":[{"metadata":{"_uuid":"0904422ddc964a3a1a043056e72b924f10cfe853"},"cell_type":"markdown","source":"# Introduction"},{"metadata":{"_uuid":"79ea5dc1bdec696128aac7586c878ea923d59e71"},"cell_type":"markdown","source":"Simple code for adding new features, applying RNN and concatinating embeddings. Additionally, some utility functions are presented.\n\nWhile analyzing other peoples work I have noticed that some parts of the code are in almost all of the kernels.  I hope this kernel will bring some freshnes and will boost your ranking! \n\nBefore going into the code, I want to acknowledge the following kerners: \n\nPreprocesing:\n\nhttps://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings\n\nModel:\n\nhttps://www.kaggle.com/shujian/different-embeddings-with-attention-fork-fork\n\nThe optimal treshold when to label the question as sincere or insincere is derived from this kernel:\n\nhttps://www.kaggle.com/eligijus/k-fold-analysis-for-the-main-code\n"},{"metadata":{"_uuid":"9493e20e848ec116febfaddea9086099eb8deb43"},"cell_type":"markdown","source":"# Input section"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"## Loading necessary packages\n\n## Basic packages:\n\nimport numpy as np \nimport pandas as pd \nimport string \nfrom tqdm import tqdm\nimport operator\nimport os\nimport gc\nimport time\nimport re\n\n## K fold analysis\n\nfrom random import shuffle\n\n## Reading embeddings:\n\nfrom gensim.models import KeyedVectors\n\n## Working with text:\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\n## Accuracy measure\n\nfrom sklearn.metrics import f1_score\n\n## Deep learning: \n\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.models import Sequential, Input\nfrom keras.layers import Dense, Dropout, LSTM, Embedding, GlobalMaxPool1D, Conv1D, MaxPooling1D \nfrom keras.layers import CuDNNLSTM, Bidirectional, CuDNNGRU, GlobalAvgPool1D, concatenate\nfrom keras.models import Model\nimport tensorflow as tf\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\n\n## Levenshtein distance\n\nimport Levenshtein ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"## Reading input\n\nd = pd.read_csv('../input/train.csv')\nY_train = d['target']\nX_train = d['question_text'] \n\n## Reading the test set \n\nd_test = pd.read_csv('../input/test.csv')\nX_test = d_test['question_text']  ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8bfe3c87b01c96c2b375a87e0be0f77bf2bdf344"},"cell_type":"markdown","source":"# Custom functions"},{"metadata":{"_uuid":"6dea56b8fd2b0cc7e972ddc14fb5b72089b54eec"},"cell_type":"markdown","source":"## Text preprocesing"},{"metadata":{"trusted":true,"_uuid":"09573952eee918ad7b63db006c31563071619a60"},"cell_type":"code","source":"def encode_digit(x):\n    \"\"\"\n    Encodes a digit in a string\n    \"\"\"\n    x = re.sub('[0-9]{5,}', ' ##### ', x)\n    x = re.sub('[0-9]{4}', ' #### ', x)\n    x = re.sub('[0-9]{3}', ' ### ', x)\n    x = re.sub('[0-9]{2}', ' ## ', x)\n    return x\n\ndef clean_digits(string_vec):\n    \"\"\"\n    Removes digits from a string vector\n    \"\"\"\n    cleaned_string = [encode_digit(s) for s in string_vec]\n    \n    return pd.Series(cleaned_string)\n\ndef clean_ws(string_vec):\n    \"\"\"\n    Cleans whitespaces\n    \"\"\"\n    cleaned_string = [re.sub( '\\s+', ' ', s).strip() for s in string_vec]\n    return pd.Series(cleaned_string)\n\ndef clean_word(char, punct):\n    \"\"\"\n    A function that removes bad punctuations and splits good ones in a given string\n    \"\"\"\n    for p in punct:\n        char = char.replace(p, f' {p} ')\n    \n    return(char)\n\ndef clean_punct(string_vec, punct):\n    \"\"\"\n    Function that cleans the punctuations\n    \"\"\"\n    cleaned_string = []\n    for char in tqdm(string_vec):\n        char = [clean_word(x, punct) for x in char.split()]\n        cleaned_string.append(' '.join(char))\n    return pd.Series(cleaned_string)   \n\ndef tokenize_text(string_vec, tokenizer, max_len):\n    \"\"\"\n    Tokenizes a given string vector\n    \"\"\"\n    token = tokenizer.texts_to_sequences(string_vec)\n    token = pad_sequences(token, maxlen = max_len)\n    \n    return token","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"205bb676b64ce7d7523e0b41b030cd9626982375"},"cell_type":"markdown","source":"## Functions for embeddings"},{"metadata":{"trusted":true,"_uuid":"fa54ba6d1b1e930d9b0da717eef1723102dab2cc"},"cell_type":"code","source":"def load_from_text(path):\n    \"\"\"\n    A functions that load embeddings from a txt document\n    \"\"\"\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(path, errors='ignore'))\n    return embeddings_index\n\ndef read_embedding(path, reading_type, binary = False):\n    \"\"\"\n    Reads the embeddings from a .txt or .vec file\n    \"\"\"\n    if(reading_type == 'text'):\n        model_embed = load_from_text(path)\n        \n    if(reading_type == 'word2vec'):\n        model_embed = KeyedVectors.load_word2vec_format(path, binary = binary)\n    \n    return model_embed  \n\ndef create_embedding_matrix(model_embed, tokenizer, max_features, embed_size):\n    \"\"\"\n    Creates the embeding matrix\n    \"\"\"\n    embedding_matrix = np.zeros((max_features, embed_size))\n    for word, index in tokenizer.word_index.items():\n        if index > max_features - 1:\n            break\n        else:\n            try:\n                embedding_matrix[index] = model_embed[word]\n            except:\n                continue\n    return embedding_matrix        \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6469749924c77441f2ef14fa44ca0e38249566b8"},"cell_type":"markdown","source":"## Text exploration functions"},{"metadata":{"trusted":true,"_uuid":"85f11c99772f886f63064b38c1f67617b9451d63"},"cell_type":"code","source":"def find_most_similar(words, model_embed, window = 1, return_top = 1):\n    \"\"\"\n    Finds the most similar words in the given embedding dictionary based on Levenshtein's distance\n    \"\"\"\n    all_keys = list(model_embed.keys())\n    key_dict = {}\n    \n    for key in all_keys:\n        key_dict.update({key: len(key)})\n    \n    vocab_mapper = []    \n    for word in tqdm(words):\n        w_l = len(word)\n        sub_key_dict = [k for k, v in key_dict.items() if w_l - window <= v <= w_l + window] # Subseting the search plane\n        dist_list = [{word : key, 'dist': Levenshtein.distance(word, key)} for key in sub_key_dict]\n        dist_list = sorted(dist_list, key = lambda k: k['dist'])[:return_top] ## Extracting the top matches \n        vocab_mapper.append(dist_list)\n    \n    ## Creating a pandas dataframe for easier exploration\n    \n    vocab = pd.DataFrame(columns = ['orig_word', 'suggestion', 'dist'])\n    \n    for entry in vocab_mapper:\n        for j in range(return_top):\n            keys = [*entry[j].keys()]\n            suggestion = entry[j][keys[0]]\n            dist = entry[j][keys[1]]\n            vocab = vocab.append({'orig_word' : keys[0], 'suggestion' : suggestion, 'dist' : dist}, ignore_index = True)\n    \n    vocab = vocab.sort_values('orig_word')\n    return vocab\n\ndef build_vocab(sentences, verbose =  True):\n    \"\"\"\n    A function that creates a vocabulary from the text\n    \"\"\"\n    vocab = {}\n    for sentence in tqdm(sentences, disable = (not verbose)):\n        for word in sentence.split():\n            try:\n                vocab[word] += 1\n            except KeyError:\n                vocab[word] = 1\n    return vocab\n\ndef check_coverage(vocab, model_embed):\n    \"\"\"\n    A function to count the words that are missing from the embeddings\n    \"\"\"\n    a = {}\n    oov = {}\n    k = 0\n    i = 0\n    for word in tqdm(vocab):\n        try:\n            a[word] = model_embed[word]\n            k += vocab[word]\n        except:\n            oov[word] = vocab[word]\n            i += vocab[word]\n            pass\n\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for  {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x\n\ndef spell_checker(string_vec, vocab_mapper):\n    \"\"\"\n    A function to change the word in a dictionary \"x\" : \"y\"\n    in the following manner: x -> y\n    \"\"\"\n    cleaned_strings = []\n    for char in string_vec:\n        cleaned_words = []\n        for x in char.split():\n            if vocab_mapper.get(x) is not None:\n                x = vocab_mapper.get(x)\n            cleaned_words.append(x)\n        cleaned_words = ' '.join(cleaned_words)\n        cleaned_strings.append(cleaned_words)\n    \n    return pd.Series(cleaned_strings)    \n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"31f8cb94f1460302268d1309769cd497bc8cbbb1"},"cell_type":"markdown","source":"## Functions for model predictions "},{"metadata":{"trusted":true,"_uuid":"28cfcc6c7426a431ec2e879d2853d91005dc8d27"},"cell_type":"code","source":"def to_binary(p_array, treshold):\n    \"\"\"\n    Converts the prediction from probability to 0 or 1\n    \"\"\"\n    y_hat = []\n    for i in range(len(p_array)):\n        if p_array[i] > treshold:\n            y_hat.append(1)\n        else:\n            y_hat.append(0)\n    return y_hat    \n\ndef optimal_treshold(y, yhat):\n    \"\"\"\n    Computes the otpimal treshold for the f1 statistic\n    \"\"\"\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i/100 for i in range(10,90)]:\n        score = f1_score(y, yhat > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2fc01b30b411d91ab2a74d20acb58cc42b11a7a4"},"cell_type":"markdown","source":"## Deep learning functions"},{"metadata":{"trusted":true,"_uuid":"7696676b64fee28e1041b2b0b52b51b0546c92b8"},"cell_type":"code","source":"def f1(y_true, y_pred):\n    '''\n    metric from here \n    https://stackoverflow.com/questions/43547402/how-to-calculate-f1-macro-in-keras\n    '''\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c683df6ea77111939750916db355da4774ca426b"},"cell_type":"markdown","source":"## Feature engineering"},{"metadata":{"trusted":true,"_uuid":"fae37e821c28926d98f070955b8ee54f19b6b0d3"},"cell_type":"code","source":"def share_unique(string_vec):\n    \"\"\"\n    A function that calculates the share of unique words in a given string\n    \"\"\"\n    share_list = []\n    for char in string_vec:\n        sh = len(set(char.split()))/len(char.split())\n        share_list.append(sh)\n    return share_list\n    \ndef share_capslock(string_vec):\n    \"\"\"\n    Calculates the share of caps locked words in a given string\n    \"\"\"\n    share_list = []\n    for char in string_vec:\n        to_upper = char.upper().split()\n        sh = len(set(char.split()).intersection(to_upper))/len(char.split())\n        share_list.append(sh)\n    return share_list    \n\ndef ends_with_symbol(string_vec, symbol):\n    \"\"\"\n    Returns a list of 1 and 0 indicating whether a string ended with a symbol or not\n    \"\"\"\n    return [int(x.endswith(symbol)) for x in string_vec]\n\ndef count_words(string_vec):\n    \"\"\"\n    Counts the number of words in a given string\n    \"\"\"\n    return [len(x.split()) for x in string_vec]\n    \ndef count_occurance(string_vec):\n    \"\"\"\n    Counts the number of ? and ! in a given string\n    \"\"\"\n    return [x.count('!') + x.count('?') for x in string_vec]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d7de9bf1d28a8563c02bca1bbd300eec55c36337"},"cell_type":"markdown","source":"# Reading the embeddings"},{"metadata":{"_uuid":"d7c6afeeffacb001b64e5c1e916ae857ecc681ad"},"cell_type":"markdown","source":"There are 4 embeddings, each of them are saved in different folders and formats:  "},{"metadata":{"trusted":true,"_uuid":"c4759018365f73e37f6a78ec39cf9d74d2076b95"},"cell_type":"code","source":"print(os.listdir(\"../input/embeddings/\"))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"edf4d6308034c45d46affbbc0f9beea33b8a1ee8"},"cell_type":"markdown","source":"A dictionary to manage them: "},{"metadata":{"trusted":true,"_uuid":"c06ac1c0b831b011aa1218a1ee2f5e6f10464b2f"},"cell_type":"code","source":"embed_dict = {'google': {'path' : '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin',\n                         'reading' : 'word2vec', \n                         'binary': True}, \n                         \n               'glove': {'path': '../input/embeddings/glove.840B.300d/glove.840B.300d.txt', \n                         'reading' : 'text',\n                         'binary': False}, \n               \n                'wiki': {'path' : '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec', \n                         'reading' : 'text', \n                         'binary': False}, \n                         \n               'paragram': {'path': '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt',\n                         'reading' : 'text', \n                          'binary' : False}\n}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"91cb353844f13af24cc939180c9d69acfcb78486"},"cell_type":"markdown","source":"We can define which embeddings we will use. If we use more than 1 then the embeddings will be concatenated. Based on empyrical results I have concluded that using 1 embedding leads to better results. Thus, we will use only the GloVe embedding.\n\nI would not recommend using more than two embeddings especially when using "},{"metadata":{"trusted":true,"_uuid":"5a6f38b9c2ccbf5fac7e90e28d99f33db00a69b9"},"cell_type":"code","source":"embeds_to_use = ['glove'] ## name of key in the 'embed_dict' dictionary","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"cd624d4f94651adb5a7101698075d280931d988e"},"cell_type":"code","source":"index = 0\nfor embed in embeds_to_use:\n    \n    print('Reading: ' + embed)\n    path = embed_dict.get(embed)['path']\n    type_of_file = embed_dict.get(embed)['reading']\n    bin = embed_dict.get(embed)['binary']\n    embedding = read_embedding(path, type_of_file, bin)\n\n    if embed == embeds_to_use[0]:\n        model_embed = embedding\n    else: \n        all_keys = [k for k in embedding.keys()]\n        index += 1\n        for key in tqdm(all_keys):\n            try:\n                vect = np.concatenate([model_embed[key], embedding[key]])\n            except:\n                vect = np.concatenate([np.zeros((1, 300 * index), dtype = 'float32')[0], embedding[key]])\n                \n            model_embed.update({key : vect})    \n            del vect\n        del all_keys    \n            \n    del embedding\n    gc.collect()\n    time.sleep(5)    ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b7c7046695141e537111a927022ea74e340776fd"},"cell_type":"markdown","source":"# Text preprocesing pipeline"},{"metadata":{"trusted":true,"_uuid":"6ee8bd0cd7a645b7ce51e64b930baef95e65d576"},"cell_type":"code","source":"## Word fixing:\n\nvocab_mapper = {'Quorans' : 'Qurans', \n                'Blockchain' : 'blockchain', \n                'blockchains' : 'blockchain',\n                'demonetisation' : 'demonetization', \n                'ethereum' : 'Ethereum', \n                'Qoura' : 'Quora', \n                'SJWs' : 'SJW', \n                'bhakts' : 'bhakti', \n                'Bhakts' : 'Bhakti', \n                'kotlin' : 'Kotlin', \n                'narcissit' : 'narcissist', \n                'Trumpism' : 'Trump', \n                'Tamilans' : 'Tamilians', \n                'acturial' : 'actuarial', \n                'demonitization' : 'demonetization', \n                'Demonetization' : 'demonetization',\n                'demonitisation' : 'demonetization',\n                'Demonetisation' : 'demonetization',\n                'Whyis' : 'Why is', \n                'AirPods' : 'AirPod', \n                'Drumpf': 'Trumpf', \n                'Zhihu' : 'Zhihua', \n                'Neuralink' : 'Neurolink', \n                'fullform' : 'full-form', \n                'biharis' : 'Biharis', \n                'madheshi' : 'Madheshi', \n                'Xiomi' : 'Xiaomi', \n                'rohingya' : 'Rohingya', \n                'Despacito' : 'Desposito', \n                'schizoids' : 'schizoid', \n                'MHTCET' : 'MHT-CET', \n                'fortnite' : 'Fortnite',\n                'Bittrex' : 'Bitrex', \n                'ReactJS' : 'JavaScript', \n                'hyperloop' : 'Hyperloop', \n                'adhaar' : 'Aadhaar', \n                'Adhaar' : 'Aadhaar', \n                'Baahubali' : 'Bahubali', \n                'Cryptocurrency' : 'cryptocurrency', \n                'cryptocurrencies' : 'cryptocurrency',\n                'cryptocoins' : 'cryptocurrency',    \n                \"\\u200b\":\" \",\n                \"\\ufeff\" : \"\",\n                \"2k17\" : '2017',\n                \"2k18\" : '2018',\n                \"nofap\": 'no fap', \n                'Brexiting' : 'Brexit',\n                'mastuburation' : 'masturbation',\n                'quara' : 'Quora',\n                'Quoras' : 'Quora',\n                \"fiancé\" : \"fiance\", \n                'π' : 'pi', \n                'Pokémon' : 'Pokemon', \n                '€' : 'euro'\n}\n\n## Expanding phrases\n\nword_expansion = {\"aren't\" : 'are not', \n                \"I'm\" : 'I am',\n                \"What's\" : 'What is',\n                \"don’t\" : \"do not\", \n                \"isn't\" : \"is not\", \n                \"I’m\" : \"I am\", \n                'aren’t' : 'are not', \n                \"Can't\" : \"cannot\", \n                \"can't\" : \"cannot\",\n                \"don't\" : 'do not', \n                \"How's\" : \"how is\", \n                \"we're\" : 'we are', \n                \"won't\" : 'will not', \n                \"they're\" : \"they are\",\n                \"he's\" : \"he is\", \n                \"doesn’t\" : \"does not\", \n                \"shouldn't\" : \"should not\", \n                \"Shouldn't\" : \"should not\",\n                \"hasn't\" : \"has not\", \n                \"couldn't\" : \"could not\", \n                \"I’ve\" : \"I have\", \n                \"aren't\" : \"are not\", \n                \"weren't\" : 'were not'}\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe679eb86aae300d91fb0bcbe2efcf0204ed0195"},"cell_type":"markdown","source":"The pipeline for preprocesing:"},{"metadata":{"trusted":true,"_uuid":"755800b4dc4a0512a12f45e7075a9037c4f03936"},"cell_type":"code","source":"## Punctuations to extract\n\npunct = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']\npunct = ''.join(punct)\n\n## Defining the preprocesing\n\ndef preproc_pipeline(string_df, model_embed, punct, word_expansion = None, vocab_mapper = None):\n    \"\"\"\n    The whole pipeline of cleaning\n    \"\"\"\n    \n    if(word_expansion is not None):\n        string_df = spell_checker(string_df, word_expansion)\n        \n    string_df = clean_punct(string_df, punct)\n    \n    if(vocab_mapper is not None):\n        string_df = spell_checker(string_df, vocab_mapper)\n    \n    string_df = clean_digits(string_df)\n    string_df = clean_ws(string_df)\n    return string_df\n\n## Applying the preprocesing\n\nX_tr = preproc_pipeline(X_train, model_embed, punct, vocab_mapper = vocab_mapper)\nX_te = preproc_pipeline(X_test, model_embed, punct, vocab_mapper = vocab_mapper)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d835729c58d2016dfff21165bba1a55c9b18aed2"},"cell_type":"markdown","source":"# Spell checking"},{"metadata":{"_uuid":"a6f847582ac566545429e5f6281c6a15755a0daa"},"cell_type":"markdown","source":"We can check which words are absent from the embeddings and use the Levenshtein distance. \n\nFinding the words that are often used in texts but not in embeddings and fixing them leads to great gains in accuracy. Different embeddings have different words, thus using the Levenshtein's distance we can augment our vocabulary to be as similar as possible to the embedding which we are using.\n\nEvery word needs to be inspected manually and then added to the dictionary 'vocab_mapper'."},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"80c6a18fb6f44d396f7b9d82f60105623f907774"},"cell_type":"code","source":"## The whole vocabulary\nvocab = build_vocab(X_tr)\n\n## A dictionary for words that are out of the vocabulary but in the embeddings\noov = check_coverage(vocab, model_embed)\n\n## Subsetting only relevant words\noov = [x[0] for x in oov if x[1] > 60]\n\n## Findint the top 5 synonyms to mannualy add them to either\n## word_expansion or vocab_mapper dictionaries\nsynonyms = find_most_similar(oov, model_embed, return_top = 3)\nprint(synonyms)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"93c67ed086b29d0e9cc8895edd4e654fcfa1b417"},"cell_type":"markdown","source":"# Hyper parameters"},{"metadata":{"trusted":true,"_uuid":"0a26ded3b22f74e8256c2450b9300fc853bb13fd"},"cell_type":"code","source":"maxlen = 70 ## Number of words in each sentence to use\nmax_features = 120000 \n\nbatch_size = 512 ## batch size\nnumb_epoch = 2 ## number of epochs\n\n## Size of the embedding\n### If we used more than one embedding, then the number of coordinates for each word is doubled, tripled, etc.\n\nembed_size = 300 * len(embeds_to_use)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70ae4c3b32ba7796e5b98e97040ae9f0825693dd"},"cell_type":"markdown","source":"# RNN model"},{"metadata":{"trusted":true,"_uuid":"e658afc9d25ca4a3e64b7a1e9662b70bfc081e33"},"cell_type":"code","source":"def RNN_model(maxlen, embed_size, max_features, embedding_matrix,\n              loss_f = 'binary_crossentropy', opti = 'adam', metr = f1):\n    \n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n    x = Bidirectional(CuDNNLSTM(128, return_sequences = True))(x)\n    x = Bidirectional(CuDNNLSTM(100, return_sequences = True))(x)\n    x = Bidirectional(CuDNNLSTM(72, return_sequences = True))(x)\n    \n    atten = Attention(maxlen)(x)\n    avg_pool = GlobalAvgPool1D()(x)\n    max_pool = GlobalMaxPool1D()(x)\n    \n    conc = concatenate([atten, avg_pool, max_pool])\n    conc = Dense(64, activation=\"relu\")(conc)\n    outp = Dense(1, activation=\"sigmoid\")(conc)   \n\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss = loss_f, optimizer = opti , metrics=[metr])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70742c52d620fa67d60adba19e587ca98784d95f"},"cell_type":"markdown","source":"## Main model creation and predictions"},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"d13209106b79c8fa6d461a8c33c1ee79769c07d5"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words = max_features)\ntokenizer.fit_on_texts(list(X_tr))\n\nX_tr = tokenize_text(X_tr, tokenizer, maxlen)\nX_te = tokenize_text(X_te, tokenizer, maxlen)   \n\n## Creating a weight matrix for words in training matrix\n\nembedding_matrix = create_embedding_matrix(model_embed, \n                                            tokenizer, \n                                            max_features, \n                                            embed_size)\n\n## Creating the model \n\nmodel = RNN_model(X_tr.shape[1], embed_size, max_features, embedding_matrix)\n\nmodel_fited = model.fit(\nX_tr,\nY_train.values, \nbatch_size = batch_size, \nnb_epoch = numb_epoch)  \n\n## Predictions\n\ny_hat_probs = model.predict(X_te)\n    \n## To binary\n\ny_hat = to_binary(y_hat_probs, 0.38)\n    \n## Creating the upload file\n\nprint('The submission file has ' + str(np.sum(y_hat) * 100/len(y_hat)) + ' percent insincere')\nd_test = d_test.reset_index()\nd_test['prediction'] = y_hat\nto_upload = d_test[['qid', 'prediction']]\nto_upload.to_csv('submission.csv', index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}