{"cells":[{"metadata":{"trusted":true,"_uuid":"ef658033c7be825232d6ad40374f45537ba4c36f"},"cell_type":"markdown","source":"* This kernel was developed to allow testing of different models, evaluation technique, embeddings etc.\n* The overall structure is original, but **the ideas came from a lot of contributions from other members.**\n* I am facing the same problem of a non determinist behavior. LB Scores are around 0.85-0.90 (the most) and one 0.95 .\n* Hope you enjoy playing with it. Let me know your results with it."},{"metadata":{"trusted":true,"_uuid":"014f67cba3b8afda767743e646aebb0b1e1a7eca","_kg_hide-input":true},"cell_type":"code","source":"def f1():\n    # Just to initialize Global Variables\n    return","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7290c5d3e608329c36a863e1527f119671d014e9"},"cell_type":"code","source":"# The idea of set_all_parameters is to have one place \n# that you can select models,embeddings, evaluation methods and parameters etc\n# in one place.","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":false,"trusted":true,"_uuid":"a46c2af9d3276ab419192aeb1f6ce05f99c5b7d5"},"cell_type":"code","source":"import time\ndef set_all_parameters():\n    \n    global epoc_used\n    global mn             # Model Number - For essembling\n    global batch_size\n    global patience\n    global nsplits\n    global stop_split\n    \n    global num_lstm\n    global num_dense\n    global rate_drop_lstm\n    global rate_drop_dense\n    global rate_drop_spatial\n    global loss\n    global act\n    global opt\n    global met\n    global es_mon\n    global es_mode\n    global Trainable\n    global lr\n    global method\n    global model_list\n    global maxlen\n    global max_features\n    global pretext_proc\n    global bstart\n\n\n    #\n    # Choose Evaluate Method - Results change using different methods\n    #\n    #Each Weight will be applied and results will be divided by the sum of weigths**\n    #method=\"0\" # Standard Fit \n    #method=\"1\" # EarlyStop F1**\n    #method=\"2\" # Manual Epochs with EarlyStop\n    #method=\"3\" # Manual Epochs with EarlyStop + CLR\n    #method=\"4\" # EarlyStop F1 + CLR*\n    #method=\"5\" # Manual Epochs\n    #method=\"5\" # EarlyStop + \n    sm =\"6\" # EarlyStop F1\n    #\n    # Possible embeddings [\"Glove\",\"Paragram\",\"Google\",\"Wiki\",\"Combined\",\"Concatenated\"]\n    # I am using Glove an Paragram only. \n    # To use Google and Wiki - search and delete comments in #embedding_matrix_2 = load_embedding(\"Wiki\",word_index)\n    #                           and #embedding_matrix_4 = load_embedding(\"Google\",word_index)\n    # Define Standard Models Parameters to be used by models.     \n    #\n    ssh = 50    #Standard Size of Hidden Layers - (Lstm or Gru)\n    ssd = 50   #Standard Size of Hidden Layers Dense \n    sdh = 0.1  #Standard drop-out rate (Lstm or Gru) \n    sdd = 0.1  #Standard drop-out rate Dense \n    sds =0.1   #Standard drop-out rate Spatial\n    ##\n    ## Model Loss\n    #  Optimizer Options loss = 'binary_crossentropy'      \n    loss_bin='binary_crossentropy'      \n    ##\n    ## Model Optimizer\n    #  Optimizer Options opt='adam' or 'rmsprop'\n    opt_adam=\"adam\"\n    opt_rmsprop=\"rmsprop\"\n    ## Model Metric\n    ## Metric Options met= [\"accuracy\"] or [f1] or [\"accuracy\",f1]  \n    #\n    met_acc=[\"accuracy\"]\n    met_f1=[f1]\n    ## Early Stop Monitor and Early Stop Mode\n    ## Early Stop Monitor Options es_mon = \"val_f1\" es_mode=\"max\"  for met=[f1]\n    #                             es_mon = \"val_acc\" es_mode=\"max\" for met=[\"accuracy\"]   \n    #                             es_mon = \"val_loss\"\n    mon_acc = \"val_acc\"\n    mode_acc = \"max\"\n    mon_f1 = \"val_f1\"\n    mode_f1 = \"max\"\n    mon_loss =\"val_loss\"\n    mode_loss=\"min\"\n    ###  4 Splits = Validation Split=0.25%\n    ###  5 Splits = Validation Split=0.20 %\n    ### 10 Splits = Validation Split=0.10 %\n    ### 12 Splits = Validation Split=0.0833 %\n    ns=4  # Standar Number of Splits\n    ### For Stratified k-fold - Validation -> nsplits=stop_splits\n    ss=4   # Stop at split stop_split ina stratified k-fold\n\n    ###\n    ### Models Parameters\n    ###\n    bs=1024    #Standard Batch Size\n    ne=8       #Standard number of epochs\n    pl=3       #Standard Patience - How many time insist running epochs after not improving metric\n\n    #Possible models [\"0\",\"1\",\"2\",\"3\",\"4\",\"5\",\"6\",\"7\",\"8\",\"9\",\"14\",\"15\",\"16\",\"17\",\"18\",\"19\"]\n    # model list parameters - does not use bellow\n    # model_list.append([model_number, weight, model_eval, embeddings,epochs,patience,batch_size, size hidden, size dense, drop hidden, drop dense, drop spatial, loss,optimizer,metric])\n   \n\n    bstart = time.time()\n    #\n    # Create the list of models that will run , at the end it essembles them with the defined weights.\n    #\n    #\n    model_list=[]\n    #model_list.append([\"18\",1,sm,\"Combine\",ne,pl,bs,ns,ss,ssh,ssd,sdh,sdd,sds,loss_bin,opt_rmsprop,met_acc,mon_acc,mode_acc])\n    # or \n    model_list.append([\"18\",1,\"6\",\"Concatenated\",10,1,1024,4,4,50,50,0.1,0.1,0.15,'binary_crossentropy','rmsprop',[\"accuracy\"],'val_acc','max'])\n    # or\n    #model_list.append([\"18\",1,\"5\",\"Combine\",ne,pl,bs,ns,ss,ssh,ssd,sdh,sdd,sds,loss_bin,opt_rmsprop,met_acc,mon_acc,mode_acc])\n    #model_list.append([\"11\",1,\"6\",\"Combine\",ne,pl,bs,ns,ss,ssh,ssd,sdh,sdd,sds,loss_bin,opt_rmsprop,met_acc,mon_loss,mode_loss])\n\n    ###\n    ###  Load Train,Val and Test \n    ###\n    maxlen = 75 # Maximum Sequence Size \n    max_features = None # Maximum Number of Words in Dictionary\n    pretext_proc=True\n    \n    ###\n    ###  Embeddings\n    ###\n    Trainable=False  # Embedding Layers trainable(True) or not(False)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"8ef12e1b193776790059a90c643ceef5d6f8c954"},"cell_type":"code","source":"set_all_parameters()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"**Import Libraries**"},{"metadata":{"trusted":true,"_uuid":"b3feca02384cc7e6b6e7043453d0615f810ef05a","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from IPython.display import display, HTML\n# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\nprint(os.listdir(\"../input/embeddings\"))\n\n# Any results you write to the current directory are saved as output.\nfrom sklearn.model_selection import train_test_split,StratifiedKFold\nfrom tqdm import tqdm\nimport math\nfrom datetime import timedelta\nimport time\nfrom datetime import datetime\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nimport colorama\nfrom colorama import Fore\n\nfrom nltk.corpus import stopwords\nfrom nltk.stem import SnowballStemmer\nfrom string import punctuation\nfrom sklearn.model_selection import train_test_split,StratifiedKFold\nfrom sklearn.metrics import roc_auc_score\nfrom sklearn import metrics\nfrom gensim.models import KeyedVectors\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Flatten, Dense, Input, LSTM, Embedding, Dropout, Activation, SpatialDropout1D, Reshape, Concatenate\nfrom keras.layers.merge import concatenate\nfrom keras.models import Model\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, Callback\nfrom keras.layers import Bidirectional, GlobalMaxPool1D,GlobalMaxPooling1D,GlobalAveragePooling1D ,Conv1D, MaxPooling1D, GRU,CuDNNLSTM,CuDNNGRU, Reshape, MaxPooling1D,AveragePooling1D\nfrom keras.optimizers import RMSprop, SGD, Nadam, Adamax, Adam\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\n#from keras import initializations\nfrom keras import initializers, regularizers, constraints\nfrom keras.layers import Conv2D, MaxPool2D\nimport keras.backend as K\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"beb6daac65cf187661115680e32ec44d2fb5913e"},"cell_type":"markdown","source":"**Attention - Keras Layer**"},{"metadata":{"trusted":true,"_uuid":"5636e8032478cd1a05b8912a943e4f035b0f17d0","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        \"\"\"\n        Keras Layer that implements an Attention mechanism for temporal data.\n        Supports Masking.\n        Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n        # Input shape\n            3D tensor with shape: `(samples, steps, features)`.\n        # Output shape\n            2D tensor with shape: `(samples, features)`.\n        :param kwargs:\n        Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n        The dimensions are inferred based on the output shape of the RNN.\n        Example:\n            model.add(LSTM(64, return_sequences=True))\n            model.add(Attention())\n        \"\"\"\n        self.supports_masking = True\n        #self.init = initializations.get('glorot_uniform')\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        # do not pass the mask to the next layers\n        return None\n\n    def call(self, x, mask=None):\n        # eij = K.dot(x, self.W) TF backend doesn't support it\n\n        # features_dim = self.W.shape[0]\n        # step_dim = x._keras_shape[1]\n\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)), K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        # apply mask after the exp. will be re-normalized next\n        if mask is not None:\n            # Cast the mask to floatX to avoid float64 upcasting in theano\n            a *= K.cast(mask, K.floatx())\n\n        # in some cases especially in the early stages of training the sum may be almost zero\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        #print weigthted_input.shape\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        #return input_shape[0], input_shape[-1]\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dd1476c152966bd94c2abef6a69c1c20dd74161","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import tensorflow as tf\nfrom keras.layers import Dense, Input, Embedding, Lambda, Dropout, Activation, SpatialDropout1D, Reshape, GlobalAveragePooling1D, merge, Flatten, Bidirectional, CuDNNGRU, add, Conv1D, GlobalMaxPooling1D\nfrom keras.layers.merge import concatenate\nfrom keras.models import Model\nfrom keras.layers.normalization import BatchNormalization\nfrom keras import optimizers\nfrom keras import initializers\nfrom keras.engine import InputSpec, Layer\nfrom keras import backend as K\n\nclass AttentionWeightedAverage(Layer):\n    \"\"\"\n    Computes a weighted average of the different channels across timesteps.\n    Uses 1 parameter pr. channel to compute the attention value for a single timestep.\n    \"\"\"\n\n    def __init__(self, return_attention=False, **kwargs):\n        self.init = initializers.get('uniform')\n        self.supports_masking = True\n        self.return_attention = return_attention\n        super(AttentionWeightedAverage, self).__init__(** kwargs)\n\n    def build(self, input_shape):\n        self.input_spec = [InputSpec(ndim=3)]\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight(shape=(input_shape[2], 1),\n                                 name='{}_W'.format(self.name),\n                                 initializer=self.init)\n        self.trainable_weights = [self.W]\n        super(AttentionWeightedAverage, self).build(input_shape)\n\n    def call(self, x, mask=None):\n        # computes a probability distribution over the timesteps\n        # uses 'max trick' for numerical stability\n        # reshape is done to avoid issue with Tensorflow\n        # and 1-dimensional weights\n        logits = K.dot(x, self.W)\n        x_shape = K.shape(x)\n        logits = K.reshape(logits, (x_shape[0], x_shape[1]))\n        ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))\n\n        # masked timesteps have zero weight\n        if mask is not None:\n            mask = K.cast(mask, K.floatx())\n            ai = ai * mask\n        att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon())\n        weighted_input = x * K.expand_dims(att_weights)\n        result = K.sum(weighted_input, axis=1)\n        if self.return_attention:\n            return [result, att_weights]\n        return result\n\n    def get_output_shape_for(self, input_shape):\n        return self.compute_output_shape(input_shape)\n\n    def compute_output_shape(self, input_shape):\n        output_len = input_shape[2]\n        if self.return_attention:\n            return [(input_shape[0], output_len), (input_shape[0], input_shape[1])]\n        return (input_shape[0], output_len)\n\n    def compute_mask(self, input, input_mask=None):\n        if isinstance(input_mask, list):\n            return [None] * len(input_mask)\n        else:\n            return None","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4c67f376f64e5f4f8efc3fe60a10fdd1cdfed87f"},"cell_type":"markdown","source":"[ https://www.kaggle.com/hireme/fun-api-keras-f1-metric-cyclical-learning-rate/code](http://)"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"d0da66dfe7a0bcee2bc195c6c53b93b983923cef"},"cell_type":"code","source":"class CyclicLR(Callback):\n    \"\"\"This callback implements a cyclical learning rate policy (CLR).\n    The method cycles the learning rate between two boundaries with\n    some constant frequency, as detailed in this paper (https://arxiv.org/abs/1506.01186).\n    The amplitude of the cycle can be scaled on a per-iteration or \n    per-cycle basis.\n    This class has three built-in policies, as put forth in the paper.\n    \"triangular\":\n        A basic triangular cycle w/ no amplitude scaling.\n    \"triangular2\":\n        A basic triangular cycle that scales initial amplitude by half each cycle.\n    \"exp_range\":\n        A cycle that scales initial amplitude by gamma**(cycle iterations) at each \n        cycle iteration.\n    For more detail, please see paper.\n    \n    # Example\n        ```python\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., mode='triangular')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```\n    \n    Class also supports custom scaling functions:\n        ```python\n            clr_fn = lambda x: 0.5*(1+np.sin(x*np.pi/2.))\n            clr = CyclicLR(base_lr=0.001, max_lr=0.006,\n                                step_size=2000., scale_fn=clr_fn,\n                                scale_mode='cycle')\n            model.fit(X_train, Y_train, callbacks=[clr])\n        ```    \n    # Arguments\n        base_lr: initial learning rate which is the\n            lower boundary in the cycle.\n        max_lr: upper boundary in the cycle. Functionally,\n            it defines the cycle amplitude (max_lr - base_lr).\n            The lr at any cycle is the sum of base_lr\n            and some scaling of the amplitude; therefore \n            max_lr may not actually be reached depending on\n            scaling function.\n        step_size: number of training iterations per\n            half cycle. Authors suggest setting step_size\n            2-8 x training iterations in epoch.\n        mode: one of {triangular, triangular2, exp_range}.\n            Default 'triangular'.\n            Values correspond to policies detailed above.\n            If scale_fn is not None, this argument is ignored.\n        gamma: constant in 'exp_range' scaling function:\n            gamma**(cycle iterations)\n        scale_fn: Custom scaling policy defined by a single\n            argument lambda function, where \n            0 <= scale_fn(x) <= 1 for all x >= 0.\n            mode paramater is ignored \n        scale_mode: {'cycle', 'iterations'}.\n            Defines whether scale_fn is evaluated on \n            cycle number or cycle iterations (training\n            iterations since start of cycle). Default is 'cycle'.\n    \"\"\"\n\n    def __init__(self, base_lr=0.001, max_lr=0.006, step_size=2000., mode='triangular',\n                 gamma=1., scale_fn=None, scale_mode='cycle'):\n        super(CyclicLR, self).__init__()\n\n        self.base_lr = base_lr\n        self.max_lr = max_lr\n        self.step_size = step_size\n        self.mode = mode\n        self.gamma = gamma\n        if scale_fn == None:\n            if self.mode == 'triangular':\n                self.scale_fn = lambda x: 1.\n                self.scale_mode = 'cycle'\n            elif self.mode == 'triangular2':\n                self.scale_fn = lambda x: 1/(2.**(x-1))\n                self.scale_mode = 'cycle'\n            elif self.mode == 'exp_range':\n                self.scale_fn = lambda x: gamma**(x)\n                self.scale_mode = 'iterations'\n        else:\n            self.scale_fn = scale_fn\n            self.scale_mode = scale_mode\n        self.clr_iterations = 0.\n        self.trn_iterations = 0.\n        self.history = {}\n\n        self._reset()\n\n    def _reset(self, new_base_lr=None, new_max_lr=None,\n               new_step_size=None):\n        \"\"\"Resets cycle iterations.\n        Optional boundary/step size adjustment.\n        \"\"\"\n        if new_base_lr != None:\n            self.base_lr = new_base_lr\n        if new_max_lr != None:\n            self.max_lr = new_max_lr\n        if new_step_size != None:\n            self.step_size = new_step_size\n        self.clr_iterations = 0.\n        \n    def clr(self):\n        cycle = np.floor(1+self.clr_iterations/(2*self.step_size))\n        x = np.abs(self.clr_iterations/self.step_size - 2*cycle + 1)\n        if self.scale_mode == 'cycle':\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(cycle)\n        else:\n            return self.base_lr + (self.max_lr-self.base_lr)*np.maximum(0, (1-x))*self.scale_fn(self.clr_iterations)\n        \n    def on_train_begin(self, logs={}):\n        logs = logs or {}\n\n        if self.clr_iterations == 0:\n            K.set_value(self.model.optimizer.lr, self.base_lr)\n        else:\n            K.set_value(self.model.optimizer.lr, self.clr())        \n            \n    def on_batch_end(self, epoch, logs=None):\n        \n        logs = logs or {}\n        self.trn_iterations += 1\n        self.clr_iterations += 1\n\n        self.history.setdefault('lr', []).append(K.get_value(self.model.optimizer.lr))\n        self.history.setdefault('iterations', []).append(self.trn_iterations)\n\n        for k, v in logs.items():\n            self.history.setdefault(k, []).append(v)\n        \n        K.set_value(self.model.optimizer.lr, self.clr())\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6776b91ff387aa9381e72f618040b7536ffc5e9c"},"cell_type":"markdown","source":"**Function to Check Embeddings Coverage over Vocabulary**"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"da3ec43762be6d06fea06c8ba551a4126314e81f"},"cell_type":"code","source":"import operator \n#for ele in t.word_counts:\n#  print(ele,t.word_counts[ele])\ndef check_coverage(vocab,embeddings_index):\n    \n    try:\n        print (\"Len - embeddings_index:\",len(embeddings_index))\n    except :\n        print ( \"Len - embeddings_index:\",len(embeddings_index.index2word))\n    print (\"Len -\",len(vocab))\n    a = {}\n    oov = {}\n    k = 0\n    i = 0\n    for word in tqdm(vocab):\n        try:\n            a[word] = embeddings_index[word]\n            k += vocab[word]\n        except:\n            \n            oov[word] = vocab[word]\n            i += vocab[word]\n            pass\n    print (\"Words Found:\",k)\n    print (\"Words Not Found:\",i)\n    print (\"Total Words:\",i+k)\n    print('Found embeddings for {:.2%} of vocab'.format(len(a) / len(vocab)))\n    print('Found embeddings for {:.2%} of all text'.format(k / (k + i)))\n    sorted_x = sorted(oov.items(), key=operator.itemgetter(1))[::-1]\n\n    return sorted_x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2c29197c4ab4544aa2e33a0b3e0d5da16bedf8a4"},"cell_type":"markdown","source":"**Capsule Layer https://www.kaggle.com/gmhost/gru-capsule**"},{"metadata":{"trusted":true,"_uuid":"820c867ebe66ba3ca58c23832ec10a111c92884d","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def squash(x, axis=-1):\n    # s_squared_norm is really small\n    # s_squared_norm = K.sum(K.square(x), axis, keepdims=True) + K.epsilon()\n    # scale = K.sqrt(s_squared_norm)/ (0.5 + s_squared_norm)\n    # return scale * x\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n    scale = K.sqrt(s_squared_norm + K.epsilon())\n    return x / scale\n\n# A Capsule Implement with Pure Keras\nclass Capsule(Layer):\n    def __init__(self, num_capsule, dim_capsule, routings=3, kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = squash\n        else:\n            self.activation = Activation(activation)\n\n    def build(self, input_shape):\n        super(Capsule, self).build(input_shape)\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(1, input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     # shape=self.kernel_size,\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n        else:\n            input_num_capsule = input_shape[-2]\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(input_num_capsule,\n                                            input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n\n    def call(self, u_vecs):\n        if self.share_weights:\n            u_hat_vecs = K.conv1d(u_vecs, self.W)\n        else:\n            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n\n        batch_size = K.shape(u_vecs)[0]\n        input_num_capsule = K.shape(u_vecs)[1]\n        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n                                            self.num_capsule, self.dim_capsule))\n        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]\n\n        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]\n        for i in range(self.routings):\n            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]\n            c = K.softmax(b)\n            c = K.permute_dimensions(c, (0, 2, 1))\n            b = K.permute_dimensions(b, (0, 2, 1))\n            outputs = self.activation(K.batch_dot(c, u_hat_vecs, [2, 2]))\n            if i < self.routings - 1:\n                b = K.batch_dot(outputs, u_hat_vecs, [2, 3])\n\n        return outputs\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c39dd7eb9d04709c3513713be5c0292c6eed3b32"},"cell_type":"markdown","source":"**Function to Pre-Process Data**"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":true,"_uuid":"6751d1c915a3910b664c1c34ba46dce81ff779ff"},"cell_type":"code","source":"########################################\n## process texts in datasets\n########################################\nimport re\n\n\ndef remove_urls (vTEXT):\n    vTEXT = re.sub(r'(https|http)?:\\/\\/(\\w|\\.|\\/|\\?|\\=|\\&|\\%)*\\b', '', vTEXT, flags=re.MULTILINE)\n    return(vTEXT)\n\ndef ReplaceThreeOrMore(s):\n    # pattern to look for three or more repetitions of any character, including\n    # newlines.\n    pattern = re.compile(r\"(.)\\1{2,}\", re.DOTALL) \n    return pattern.sub(r\"\\1\", s)\n\ndef splitstring(s):\n    # searching the number of characters to split on\n    proposed_pattern = s[0]\n    for i, c in enumerate(s[1:], 1):\n        if c != \" \":\n            if proposed_pattern == s[i:(i+len(proposed_pattern))]:\n                # found it\n                break\n            else:\n                proposed_pattern += c\n    else:\n        exit(1)\n\n    return proposed_pattern\n\ndef clean_numbers(x):\n\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n#Regex to remove all Non-Alpha Numeric and space\nspecial_character_removal=re.compile(r'[^a-z\\d ]',re.IGNORECASE)\n\n#regex to replace all numerics\nreplace_numbers=re.compile(r'\\d+',re.IGNORECASE)\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium'\n\n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)\n\ndef clean_text(x):\n\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x\n\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\n\ndef clean_text_NOTUSED(x):\n\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\ndef text_to_wordlist(text,to_lower=True, rem_urls=False, rem_3plus=False,\n                     clean_t=True, clean_num=True,mispelling=True,rem_specwords= False,\n                     split_repeated=True, rem_special=False, rep_num=False, \n                     man_adj=True, rem_stopwords=False, stem_snowball=False,\n                     stem_porter=False, lemmatize=False):\n\n    # Clean the text, with the option to remove stopwords and to stem words.\n    \n    # Convert words to lower case and split them\n    if rem_urls:\n        text = remove_urls(text)\n    if to_lower:    \n        text = text.lower()\n    if rem_3plus:    \n        text = ReplaceThreeOrMore(text)\n        \n    if clean_t:\n        text= clean_text(text)\n        \n    if clean_num:\n        text= clean_numbers(text)    \n        \n    if mispelling:\n        text= replace_typical_misspell(text)\n\n    if man_adj: \n        # Clean the text\n        text = re.sub(r\"[^A-Za-z0-9^,!.\\/'+-=]\", \" \", text)\n        text = re.sub(r\"what's\", \"what is \", text)\n        text = re.sub(r\"\\'s\", \" \", text)\n        text = re.sub(r\"\\'ve\", \" have \", text)\n        text = re.sub(r\"can't\", \"cannot \", text)\n        text = re.sub(r\"n't\", \" not \", text)\n        text = re.sub(r\"i'm\", \"i am \", text)\n        text = re.sub(r\"\\'re\", \" are \", text)\n        text = re.sub(r\"\\'d\", \" would \", text)\n        text = re.sub(r\"\\'ll\", \" will \", text)\n        text = re.sub(r\",\", \" \", text)\n        text = re.sub(r\"\\.\", \" \", text)\n        text = re.sub(r\"!\", \" ! \", text)\n        text = re.sub(r\"\\/\", \" \", text)\n        text = re.sub(r\"\\^\", \" ^ \", text)\n        text = re.sub(r\"\\+\", \" + \", text)\n        text = re.sub(r\"\\-\", \" - \", text)\n        text = re.sub(r\"\\=\", \" = \", text)\n        text = re.sub(r\"'\", \" \", text)\n        text = re.sub(r\"(\\d+)(k)\", r\"\\g<1>000\", text)\n        text = re.sub(r\":\", \" : \", text)\n        text = re.sub(r\" e g \", \" eg \", text)\n        text = re.sub(r\" b g \", \" bg \", text)\n        text = re.sub(r\" u s \", \" american \", text)\n        text = re.sub(r\"\\0s\", \"0\", text)\n        text = re.sub(r\" 9 11 \", \"911\", text)\n        text = re.sub(r\"e - mail\", \"email\", text)\n        text = re.sub(r\"j k\", \"jk\", text)\n        text = re.sub(r\"\\s{2,}\", \" \", text)\n\n    # split them into a list\n    text = text.split()\n    \n    if split_repeated:\n        for i, c in enumerate(text):\n            text[i]=splitstring(c)\n    \n    if rem_specwords:    \n        to_remove = ['a','to','of','and']\n        text = [w for w in text if not w in to_remove]\n        \n    # Optionally, remove stop words\n    if rem_stopwords:\n        stops = set(stopwords.words(\"english\"))\n        text = [w for w in text if not w in stops]\n    \n    text = \" \".join(text)\n    \n    #Remove Special Characters\n    if rem_special: \n        text=special_character_removal.sub('',text)\n    \n    #Replace Numbers\n    if rep_num:     \n        text=replace_numbers.sub('n',text)\n\n    # Optionally, shorten words to their stems\n    if stem_snowball:\n        text = text.split()\n        stemmer = SnowballStemmer('english')\n        stemmed_words = [stemmer.stem(word) for word in text]\n        text = \" \".join(stemmed_words)\n    \n    if stem_porter:\n        st = PorterStemmer()\n        txt = \" \".join([st.stem(w) for w in text.split()])\n        \n    if lemmatize:\n        wordnet_lemmatizer = WordNetLemmatizer()\n        txt = \" \".join([wordnet_lemmatizer.lemmatize(w) for w in text.split()])   \n \n    # Return a list of words\n    return(text)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"56e498e0476cff4ac9c4426154ab23506e5f5e27"},"cell_type":"markdown","source":"**Save Tokenizer for future use**"},{"metadata":{"trusted":true,"_uuid":"06f83064d4976d47662a89920a634dc2e327d82b","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import pickle\ndef save_tokenizer( file, tokenizer):\n    # saving\n    with open(file, 'wb') as handle:\n        pickle.dump(tokenizer, handle, protocol=pickle.HIGHEST_PROTOCOL)\n\ndef load_tokenizer( file, tokenizer):\n    # loading\n    with open(file, 'rb') as handle:\n        tokenizer = pickle.load(handle)\n    return tokenizer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"44985474cd00db22b335f85c3ee65e74eb5f0fab"},"cell_type":"markdown","source":"**Load Train and Tests Tables, Build the Vocbulary**\n**Optional: Do Text Processing to clean data**"},{"metadata":{"trusted":true,"_uuid":"e1eb8017abbbfb1c92e9efe37c607114c6a92640","_kg_hide-input":true},"cell_type":"code","source":"def load_and_prec(PreProcess=False):\n    global max_features \n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n    ## split to train and val\n    #train_fit, train_val = train_test_split(train_df, test_size=0.08, random_state=2018)\n    train_X=train_df[\"question_text\"].values\n    test_X=test_df[\"question_text\"].values\n    \n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    if PreProcess:\n        train_questions = []\n        for text in train_X:\n            train_questions.append(text_to_wordlist(text))  \n        test_questions=[]\n        for text in test_X:\n            test_questions.append(text_to_wordlist(text)) \n        tokenizer.fit_on_texts(train_questions+test_questions)   \n        train_X = tokenizer.texts_to_sequences(train_questions)\n        test_X = tokenizer.texts_to_sequences(test_questions) \n    else:\n        tokenizer.fit_on_texts(list(train_X)+list(test_X))\n        train_X = tokenizer.texts_to_sequences(train_X)\n        test_X = tokenizer.texts_to_sequences(test_X)\n\n    print(len(train_X), 'train sequences')\n    print(len(test_X), 'test sequences')\n    print('Average train sequence length: {}'.format(np.mean(list(map(len, train_X)), dtype=int)))\n    print('Average test sequence length: {}'.format(np.mean(list(map(len, test_X)), dtype=int)))\n    print('Max train sequence length: {}'.format(np.max(list(map(len, train_X)))))\n    print('Max test sequence length: {}'.format(np.max(list(map(len, test_X)))))  \n    \n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n  \n    print(len(tokenizer.word_index))\n    print(len(tokenizer.word_counts))\n    if (max_features==None):\n        max_features = len(tokenizer.word_index)+1\n    #save_tokenizer('tokenizer.pickle',tokenizer)\n    return train_X, test_X, train_y, test_df, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"005f75c413eb333af80ac8c6ab03032d59d8cc92"},"cell_type":"markdown","source":"**mdln=\"0-Bidirec(CuDNNGRU)-GlobalMax-Dense-Dropout\"**"},{"metadata":{"trusted":true,"_uuid":"2e126f1345f474f4a0edae8fc12b26fa30a06b4f","_kg_hide-input":true},"cell_type":"code","source":"########################################\n## BASE Model\n########################################\ndef create_model0(embedding_matrix):\n\n    mdln=\"0-Bidirec(CuDNNGRU)-GlobalMax-Dense-Dropout\"\n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n    \n    x = LSTM(num_lstm, return_sequences=True)(embedding_layer)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(num_dense, activation=\"relu\")(x)\n    x = Dropout(rate_drop_dense)(x)\n    preds = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)\n       \n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"12e3dde366494a307484b79125d52e59480fe085"},"cell_type":"markdown","source":"**mdln=\"1-LSTM-Dropout-Attention-Dense-Dropout-BatchNormalization\"**"},{"metadata":{"trusted":true,"_uuid":"b07c5ef2f56dfab25d66daddc798ac32519f0d94","_kg_hide-input":true},"cell_type":"code","source":"def create_model1(embedding_matrix):\n    \n    mdln=\"1-LSTM-Dropout-Attention-Dense-Dropout-BatchNormalization\"\n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n\n    x = LSTM(num_lstm, dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm,return_sequences=True)(embedding_layer)\n    x = Dropout(rate_drop_lstm)(x)\n    x = Attention(maxlen)(x)\n    x = Dense(num_dense, activation=\"relu\")(x)\n    x = Dropout(rate_drop_dense)(x)\n    x = BatchNormalization()(x)\n    preds = Dense(1, activation='sigmoid')(x)\n\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)\n    \n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3a4a23706ae841cdb10f856df491f21f8d3b924c"},"cell_type":"markdown","source":"**    mdln=\"2-Bidirect(CuDNNLSTM)-GlobalMax-Dense-Dropout-Dense-Dropout\"**"},{"metadata":{"trusted":true,"_uuid":"e8dfe977c735e8d2b3ac903783b8a0c1588565dd","_kg_hide-input":true},"cell_type":"code","source":"########################################\n## BASE Model\n########################################\ndef create_model2(embedding_matrix):\n\n    mdln=\"2-Bidirect(CuDNNLSTM)-GlobalMax-Dense-Dropout-Dense-Dropout\"\n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n\n    #x = Bidirectional(LSTM(num_dense, return_sequences=True, dropout=rate_drop_dense, recurrent_dropout=rate_drop_lstm))(embedding_layer)\n    x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(num_dense, activation=\"relu\")(x)\n    x = Dropout(rate_drop_dense)(x)\n    x = Dense(num_dense, activation=\"relu\")(x)\n    x = Dropout(rate_drop_dense)(x)\n    preds = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)\n    \n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"43ea927b3b2a2a2f455181336cf86bf94385f4e2"},"cell_type":"markdown","source":"**  mdln=\"3-Bidirect(CuDNNLSTM)-Bidirect(CuDNNLSTM)-Attention-Dense\"**"},{"metadata":{"trusted":true,"_uuid":"aa5575b1c75a517d9b7e89834838f6238ab91e04","_kg_hide-input":true},"cell_type":"code","source":"def create_model3(embedding_matrix):\n    \n    mdln=\"3-Bidirect(CuDNNLSTM)-Bidirect(CuDNNLSTM)-Attention-Dense\"\n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n    num_lstm2=int(num_lstm/2)\n    x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)\n    x = Bidirectional(CuDNNLSTM(num_lstm2, return_sequences=True))(x)\n    x = Attention(maxlen)(x)\n    x = Dense(num_dense, activation=\"relu\")(x)\n    preds = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(lr=1e-3),metrics=met)\n\n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"02871cdaccb6f6bd7befe5174dd9e6abd93f8678"},"cell_type":"markdown","source":"**mdln=\"4-Reshape-Concat(Conv2D(Filters)+MaxPool2d)-Flatten-Dropout\"**"},{"metadata":{"trusted":true,"_uuid":"09abaaf9a9709dd9b76101d92a98b8fb9d3daafe","_kg_hide-input":true},"cell_type":"code","source":"def create_model4( embedding_matrix):\n    \n    mdln=\"4-Reshape-Concat(Conv2D(Filters)+MaxPool2d)-Flatten-Dropout\"\n    \n    filter_sizes = [1,2,3,5]\n    num_filters = 36\n\n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n    x = Reshape((maxlen, embedding_matrix.shape[1], 1))(embedding_layer)\n    maxpool_pool = []\n    for i in range(len(filter_sizes)):\n        conv = Conv2D(num_filters, kernel_size=(filter_sizes[i], embedding_matrix.shape[1]),\n                                     kernel_initializer='he_normal', activation='elu')(x)\n        maxpool_pool.append(MaxPool2D(pool_size=(maxlen - filter_sizes[i] + 1, 1))(conv))\n\n    z = Concatenate(axis=1)(maxpool_pool)   \n    z = Flatten()(z)\n    z = Dropout(0.1)(z)\n\n    preds = Dense(1, activation=\"sigmoid\")(z)\n\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)\n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2feeab8def24aceba6ed5615995a802f4545ecfd"},"cell_type":"markdown","source":"** mdln=\"05-conv-max-conv-max-bulstm-max-dd\"**"},{"metadata":{"trusted":true,"_uuid":"84ad447bac909c27bf0e8b005f3426d1921f5ae5","_kg_hide-input":true},"cell_type":"code","source":"from keras.models import Sequential\n\ndef create_model5(embedding_matrix):\n\n    mdln=\"05-conv-max-conv-max-bulstm-max-dd\"\n    \n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n\n    #x = Dropout(0.2)(embedded_sequences)\n    #x = Conv1D(filters=64, kernel_size=5, padding='same', activation='relu')(x)\n    x = Dropout(0.2)(embedding_layer)\n    x = Conv1D(filters=embedding_matrix.shape[1], kernel_size=4, padding='same', activation='relu')(x)\n    #x = BatchNormalization()(x)\n    x = MaxPooling1D(pool_size=2)(x)\n    x = Conv1D(filters=embedding_matrix.shape[1], kernel_size=4, padding='same', activation='relu')(x)\n    #x = BatchNormalization()(x)\n    x = MaxPooling1D(pool_size=2)(x)\n    #x = GRU(32)(main)\n    #x = Dense(32, activation=\"relu\")(x)\n    #x = BatchNormalization()(x)\n    #x = MaxPooling1D(pool_size=4)(x)\n    #x = LSTM(num_lstm, dropout=rate_drop_lstm, recurrent_dropout=rate_drop_lstm,return_sequences=True)(x)\n    x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(x)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(num_dense, activation=\"relu\")(x)\n    x = Dropout(rate_drop_dense)(x)\n    x = Dense(num_dense, activation=\"relu\")(x)\n    x = Dropout(rate_drop_dense)(x)\n    preds = Dense(1, activation='sigmoid')(x)\n\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)\n\n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8eddec6db939857d177014b7a898a47b73b5bd04"},"cell_type":"markdown","source":"** mdln=\"6-Bidirect(CuDNNGRU)-Cocat(GlobalMax+Global Avarage)- Dense - Dropout\"**"},{"metadata":{"trusted":true,"_uuid":"6262edb16fefa1d1d00b783f9e265188d3fc18e0","_kg_hide-input":true},"cell_type":"code","source":"def create_model6( embedding_matrix):\n\n    mdln=\"6-Bidirect(CuDNNGRU)-Cocat(GlobalMax+Global Avarage)- Dense - Dropout\"\n\n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n\n\n    x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)\n  \n\n    tower_1 = GlobalMaxPool1D()(x)\n    tower_2 = GlobalAveragePooling1D()(x)\n    \n    output = concatenate([  tower_1, tower_2])\n\n    x = Dense(num_dense, activation=\"relu\")(output)\n    x = Dropout(rate_drop_dense)(x)\n    preds = Dense(1, activation=\"sigmoid\")(x)                         \n\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy',optimizer=opt,metrics=met)\n\n   \n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c56e849cf4b1e9fdd73b60421f5aeb108e4f1163"},"cell_type":"markdown","source":"**mdln=\"7-Bidirect(CuDNNGRU)-Attention-Dense-Dropout\"**"},{"metadata":{"trusted":true,"_uuid":"bb6aae7402eac36ca50473a1f6e204d0fcc98af2","_kg_hide-input":true},"cell_type":"code","source":"def create_model7( embedding_matrix):\n    \n    mdln=\"7-Bidirect(CuDNNGRU)-Attention-Dense-Dropout\"\n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n    x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)\n    x = Attention(maxlen)(x) # New\n    x = Dense(num_dense, activation=\"relu\")(x)\n    x = Dropout(rate_drop_dense)(x)\n    preds = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)\n    \n    return model , mdln   ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a3cbab1a8efbca2d85407e8a66d8c3a24751c6af"},"cell_type":"markdown","source":"** mdln=\"8-Bidirect(CuDNNGRU)-Concat(GlobalMax+GlobalAverage)-Dense-Dropout\"**"},{"metadata":{"trusted":true,"_uuid":"8caca32563ad182819548b17342776e1c86464aa","_kg_hide-input":true},"cell_type":"code","source":"def create_model8( embedding_matrix):\n    \n    mdln=\"8-Bidirect(CuDNNGRU)-Concat(GlobalMax+GlobalAverage)-Dense-Dropout\"    \n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n    x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    conc = concatenate([avg_pool, max_pool])\n    conc = Dense(num_dense, activation=\"relu\")(conc)\n    conc = Dropout(rate_drop_dense)(conc)\n    preds = Dense(1, activation=\"sigmoid\")(conc)\n    \n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)\n    \n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"090b549ef0a7ed18b38f67af9be9eecf72ce3449"},"cell_type":"markdown","source":"**    mdln=\"9-Bidirect(CuDNNGRU)-Bidirect(CuDNNGRU)-Bidirect(CuDNNGRU)-Attention-Dense\"**"},{"metadata":{"trusted":true,"_uuid":"c541eaa64e6dab5ff96f8f367317034fdc0e30d2","_kg_hide-output":false,"_kg_hide-input":true},"cell_type":"code","source":"def create_model9(embedding_matrix):\n    mdln=\"9-Bidirect(CuDNNGRU)-Bidirect(CuDNNGRU)-Bidirect(CuDNNGRU)-Attention-Dense\"\n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n    num_lstm2=int(num_lstm)\n    num_lstm12=num_lstm2+int(num_lstm2/2)\n    x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)\n    x = Bidirectional(CuDNNGRU(num_lstm12, return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(num_lstm2, return_sequences=True))(x)\n    x = Attention(maxlen)(x)\n    preds = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)\n    \n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b348ee938d52b4d28ed5e3b142a082a4de4728d2"},"cell_type":"markdown","source":"** mdln=\"10-Spatial-Bidirect(CuDNNGRU)-conv2-maxpool-attavr-average-concatenate-drop-dense**"},{"metadata":{"trusted":true,"_uuid":"5bf3754ee4e5d26bee0e1a0b044ac50d2cc031c8","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def create_model10(embedding_matrix):\n\n        mdln=\"10-Spatial-Bidirect(CuDNNGRU)-conv2-maxpool-attavr-average-concatenate-drop-dense\"      \n        input_layer = Input(shape=(maxlen,))\n        embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n\n    \n        recurrent_units = 64\n        filter1_nums = 128\n\n        embedding_layer = SpatialDropout1D(rate_drop_spatial)(embedding_layer)\n        rnn_1 = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(embedding_layer)\n\n        #conv_1 = Conv1D(filter1_nums, 1, kernel_initializer=\"uniform\", padding=\"valid\", activation=\"relu\", strides=1)(rnn_1)\n        #maxpool = GlobalMaxPooling1D()(conv_1)\n        #attn = AttentionWeightedAverage()(conv_1)\n        #average = GlobalAveragePooling1D()(conv_1)\n\n        conv_2 = Conv1D(filter1_nums, 2, kernel_initializer=\"normal\", padding=\"valid\", activation=\"relu\", strides=1)(rnn_1)\n        maxpool = GlobalMaxPooling1D()(conv_2)\n        attn = AttentionWeightedAverage()(conv_2)\n        average = GlobalAveragePooling1D()(conv_2)\n\n        concatenated = concatenate([maxpool, attn, average], axis=1)\n        x = Dropout(rate_drop_dense)(concatenated)\n        x = Dense(num_dense, activation=\"relu\")(x)\n        output_layer = Dense(1, activation=\"sigmoid\")(x)\n\n        model = Model(inputs=input_layer, outputs=output_layer)\n        adam_optimizer = optimizers.Adam(lr=1e-3, clipvalue=5, decay=1e-5)\n        model.compile(loss='binary_crossentropy', optimizer=adam_optimizer, metrics=met)\n        return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3d2dc6c0aa1de4a401ac56760419fa0a8dfb1308"},"cell_type":"markdown","source":"**mdln=\"11-Capsule\"  ** https://www.kaggle.com/gmhost/gru-capsule"},{"metadata":{"trusted":true,"_uuid":"000168398d9688b5f5335df65204b891efcc398c","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from keras.initializers import *\ndef create_model11(embedding_matrix):\n\n        mdln=\"11-Capsule\"      \n        input_layer = Input(shape=(maxlen,))\n        embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n\n \n        x = SpatialDropout1D(rate=rate_drop_spatial)(embedding_layer)\n        x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True, \n                                    kernel_initializer=glorot_normal(seed=123000), recurrent_initializer=orthogonal(gain=1.0, seed=10000)))(x)\n\n        x = Capsule(num_capsule=10, dim_capsule=10, routings=4, share_weights=True)(x)\n\n        x = Flatten()(x)\n\n        x = Dense(num_dense, activation='relu', kernel_initializer=glorot_normal(seed=123000))(x)\n        x = Dropout(rate_drop_dense)(x)\n        x = BatchNormalization()(x)\n\n        preds = Dense(1, activation=\"sigmoid\")(x)\n        model = Model(inputs=input_layer, outputs=preds)\n        model.compile(loss='binary_crossentropy', optimizer=Adam(),)\n        return model,mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae9996fca79366eb7160181ff979c4ac4810ff2d"},"cell_type":"markdown","source":""},{"metadata":{"trusted":true,"_uuid":"5fb2dc715926700a6b7a7d5789a7e5427bd9ea7d","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def create_model12(embedding_matrix):\n    \n    mdln=\"12-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout\"\n       \n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n    x = SpatialDropout1D(rate_drop_spatial)(embedding_layer)\n    x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(x)\n    y = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)\n    \n    atten_1 = AttentionWeightedAverage()(x) # skip connect\n    atten_2 = AttentionWeightedAverage()(y)\n    avg_pool = GlobalAveragePooling1D()(y)\n    max_pool = GlobalMaxPooling1D()(y)\n    \n    conc = concatenate([atten_1, atten_2, avg_pool, max_pool])\n    conc = Dense(num_dense, activation=\"relu\")(conc)\n    conc = Dropout(rate_drop_dense)(conc)\n    preds = Dense(1, activation=\"sigmoid\")(conc)    \n\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)\n    \n    return model,mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"236ca6d709bf1846d389aefd0985c8cbb3a32bdb"},"cell_type":"markdown","source":"**  mdln=\"13-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout\"**\nSame as 18, but using initiaizers to test deterministic behavior"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"20f3d531690261a75ccfcac9268176fa3cc5aaf4"},"cell_type":"code","source":"from keras.initializers import he_normal, he_uniform,  glorot_normal,  glorot_uniform\n\ndef create_model13(embedding_matrix):\n    \n    mdln=\"13-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout\"\n       \n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n    x = SpatialDropout1D(rate_drop_spatial)(embedding_layer)\n    x = Bidirectional(CuDNNLSTM(num_lstm, kernel_initializer=glorot_uniform(seed = 2018), return_sequences=True))(x)\n    y = Bidirectional(CuDNNGRU(num_lstm,  kernel_initializer=glorot_uniform(seed = 2018),return_sequences=True))(x)\n    \n    atten_1 = Attention(maxlen)(x) # skip connect\n    atten_2 = Attention(maxlen)(y)\n    avg_pool = GlobalAveragePooling1D()(y)\n    max_pool = GlobalMaxPooling1D()(y)\n    \n    conc = concatenate([atten_1, atten_2, avg_pool, max_pool])\n    conc = Dense(num_dense, kernel_initializer=he_uniform(seed=2018), activation=\"relu\")(conc)\n    conc = Dropout(rate_drop_dense)(conc)\n    preds = Dense(1, activation=\"sigmoid\")(conc)    \n\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)\n    \n    return model,mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fd9c3bb1dd9035cc523d0bcacbd2fc7ab5e8a5b1"},"cell_type":"markdown","source":"** mdln=\"14-Bidirect(CuDNNLSTM)-SpatialDropout-GlobalMax-BatchNorm-Dense-Dropout\" **"},{"metadata":{"trusted":true,"_uuid":"5f9c5822e4f85bb061bfc0c83413e4458c922831","_kg_hide-input":true},"cell_type":"code","source":"def create_model14(embedding_matrix):\n\n        mdln=\"14-Bidirect(CuDNNLSTM)-SpatialDropout-GlobalMax-BatchNorm-Dense-Dropout\"      \n        input_layer = Input(shape=(maxlen,))\n        embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n\n        # we add a GlobalMaxPool1D, which will extract information from the embeddings\n        # of all words in the document\n        x = CuDNNLSTM(num_lstm, return_sequences=True)(embedding_layer)\n        x = SpatialDropout1D(rate_drop_spatial)(x)\n        x = GlobalMaxPool1D()(x)\n\n        # normalized dense layer followed by dropout\n        x = BatchNormalization()(x)\n        x = Dense(num_dense)(x)\n        x = Dropout(rate_drop_dense)(x)\n\n        # We project onto a six-unit output layer, and squash it with sigmoids:\n        preds = Dense(1, activation='sigmoid')(x)\n\n        model = Model(inputs=input_layer, outputs=preds)\n        model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)\n                                         \n        \n        return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"563862c8bcade5d3c16eb74d4bcfbae66792cb63"},"cell_type":"markdown","source":"**mdln=\"15-SpatialDropout-Bidirect(CudNNGRU)+Concat(GlobalMax+GlobalAverage)+Dense+Dropout\"**"},{"metadata":{"trusted":true,"_uuid":"eb1f42115ebdcb66739d38fd153dedcc85ccfc78","_kg_hide-input":true},"cell_type":"code","source":"def create_model15(embedding_matrix):\n### https://www.kaggle.com/yekenot/pooled-gru-fasttext\n        mdln=\"15-SpatialDropout-Bidirect(CudNNGRU)+Concat(GlobalMax+GlobalAverage)+Dense+Dropout\"\n        input_layer = Input(shape=(maxlen,))\n        embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n   \n        x = SpatialDropout1D(rate_drop_spatial)(embedding_layer)\n        x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)\n        avg_pool = GlobalAveragePooling1D()(x)\n        max_pool = GlobalMaxPooling1D()(x)        \n        conc = concatenate([avg_pool, max_pool])\n        x = Dense(num_dense, activation=\"relu\")(conc)\n        x = Dropout(rate_drop_dense)(x)\n        preds = Dense(1, activation=\"sigmoid\")(x)\n\n        model = Model(inputs=input_layer, outputs=preds)\n        model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)\n     \n        return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c207ad3de5e11615f2b43d15c904fad29c8ffbb7"},"cell_type":"markdown","source":"** mdln=\"16-Bidirect(CudNNGRU)+GlobalMax+Dense+Dropout\"**"},{"metadata":{"trusted":true,"_uuid":"fd6a40cb3c3ab55d51b0dc8adf40b839c607c6e3","_kg_hide-input":true},"cell_type":"code","source":"def create_model16(embedding_matrix):\n\n    mdln=\"16-Bidirect(CudNNGRU)+GlobalMax+Dense+Dropout\"\n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n\n    x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)\n    x = GlobalMaxPool1D()(x)\n    x = Dense(num_dense, activation=\"relu\")(x)\n    x = Dropout(rate_drop_dense)(x)\n    preds = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)\n    \n    return model, mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a2ede37b7ff8a7685fa89435e133884c30649b5"},"cell_type":"markdown","source":"**        mdln=\"17-Bidirec(CuDNNLSTM)-GlobalMax-Dense-Dropout-Dense-Dropout\"**"},{"metadata":{"trusted":true,"_uuid":"6d4115ce49450448f336c08dd989c29437258cba","_kg_hide-input":true},"cell_type":"code","source":"def create_model17(embedding_matrix ):\n    \n        mdln=\"17-Bidirec(CuDNNLSTM)-GlobalMax-Dense-Dropout-Dense-Dropout\"\n        input_layer = Input(shape=(maxlen,))\n        embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n        x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(embedding_layer)\n        x = GlobalMaxPool1D()(x)\n        x = Dense(num_dense, activation=\"relu\")(x)\n        x = Dropout(rate_drop_dense)(x)\n        x = Dense(num_dense, activation=\"relu\")(x)\n        x = Dropout(rate_drop_dense)(x)\n        preds = Dense(1, activation=\"sigmoid\")(x)\n        model = Model(inputs=input_layer, outputs=preds)\n        model.compile(loss='binary_crossentropy',optimizer=opt, metrics=met)\n\n        return model, mdln  ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5759914de28ed8411613b61fb39fb517183832c"},"cell_type":"markdown","source":"**mdln=\"18-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout\""},{"metadata":{"trusted":true,"_uuid":"4515901ab169477144ecf37f2ff0d85b02f4d482","_kg_hide-input":true},"cell_type":"code","source":"def create_model18(embedding_matrix):\n    \n    mdln=\"18-SpatialDropout+Bidirect(CudNNGRU)+Bidirect(CudNLSTM)+Attention+Attention+Dense+Dropout\"\n       \n    input_layer = Input(shape=(maxlen,))\n    embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                weights=[embedding_matrix], trainable=Trainable)(input_layer)\n    x = SpatialDropout1D(rate_drop_spatial)(embedding_layer)\n    x = Bidirectional(CuDNNLSTM(num_lstm, return_sequences=True))(x)\n    y = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)\n    \n    atten_1 = Attention(maxlen)(x) # skip connect\n    atten_2 = Attention(maxlen)(y)\n    avg_pool = GlobalAveragePooling1D()(y)\n    max_pool = GlobalMaxPooling1D()(y)\n    \n    conc = concatenate([atten_1, atten_2, avg_pool, max_pool])\n    conc = Dense(num_dense, activation=\"relu\")(conc)\n    conc = Dropout(rate_drop_dense)(conc)\n    preds = Dense(1, activation=\"sigmoid\")(conc)    \n\n    model = Model(inputs=input_layer, outputs=preds)\n    model.compile(loss='binary_crossentropy', optimizer=opt, metrics=met)\n    \n    return model,mdln","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"477b8adc1aea1e0a791fe3b8806826eb86fe4bc4"},"cell_type":"markdown","source":"**mdln=\"19-SpatialDropout+Bidirect(CudNNGRU)+Conv1D+Concat(GlobalMax+GlobalAverage)+Dense+Dropout\"\n**"},{"metadata":{"trusted":true,"_uuid":"9abe04a86352175327b802f2e5fe22ec0bc5a847","_kg_hide-input":true},"cell_type":"code","source":"def create_model19( embedding_matrix):\n    \n    \n        mdln=\"19-SpatialDropout+Bidirect(CudNNGRU)+Conv1D+Concat(GlobalMax+GlobalAverage)+Dense+Dropout\"\n        \n        input_layer = Input(shape=(maxlen,))\n        embedding_layer = Embedding(embedding_matrix.shape[0], embedding_matrix.shape[1],\n                                    weights=[embedding_matrix], trainable=Trainable)(input_layer)\n\n        x = SpatialDropout1D(rate_drop_spatial)(embedding_layer)\n        x = Bidirectional(CuDNNGRU(num_lstm, return_sequences=True))(x)\n\n        #x = Conv1D(filters=num_lstm, kernel_size=2, padding='same', activation='relu')(x)\n        x = Conv1D(64, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(x)\n        tower_1 = GlobalMaxPool1D()(x)\n        tower_2 = GlobalAveragePooling1D()(x)\n\n        output = concatenate([  tower_1, tower_2])\n\n        out = Dense(num_dense, activation=\"relu\")(output)\n        out = Dropout(rate_drop_dense)(out)\n        preds = Dense(1, activation=\"sigmoid\")(out)                         \n\n        model = Model(inputs=input_layer, outputs=preds)\n        #model.compile(loss='binary_crossentropy', optimizer=opt, metrics=['accuracy'])\n        model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-3),metrics=met)               \n\n\n        return model, mdln    \n        \n       ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b99687b563b608724ab9d96fac76ca25f6f907dd"},"cell_type":"markdown","source":"**Auxiliary Funciton to Create the Models**"},{"metadata":{"trusted":true,"_uuid":"92b899ad0ae295e990a17d6b7415226a2c92c0de","_kg_hide-input":true},"cell_type":"code","source":"def create_model (label,embedding_matrix):\n    \n            if (label==\"0\"):\n                return create_model0(embedding_matrix)\n            if (label==\"1\"):\n                return create_model1(embedding_matrix)\n            if (label==\"2\"):\n                return create_model2(embedding_matrix)\n            if (label==\"3\"):\n                return create_model3(embedding_matrix)            \n            if (label==\"4\"):\n                return create_model4(embedding_matrix)\n            if (label==\"5\"):\n                return create_model5(embedding_matrix)\n            if (label==\"6\"):\n                return create_model6(embedding_matrix)\n            if (label==\"7\"):\n                return create_model7(embedding_matrix)  \n            if (label==\"8\"):\n                return create_model8(embedding_matrix)\n            if (label==\"9\"):\n                return create_model9(embedding_matrix) \n            if (label==\"10\"):\n                return create_model10(embedding_matrix)            \n            if (label==\"11\"):\n                return create_model11(embedding_matrix)                \n            if (label==\"12\"):\n                return create_model12(embedding_matrix)                \n            if (label==\"13\"):\n                return create_model13(embedding_matrix)                \n            if (label==\"14\"):\n                return create_model14(embedding_matrix)                \n            if (label==\"15\"):\n                return create_model15(embedding_matrix)                \n            if (label==\"16\"):\n                return create_model16(embedding_matrix)   \n            if (label==\"17\"):\n                return create_model17(embedding_matrix) \n            if (label==\"18\"):\n                return create_model18(embedding_matrix) \n            if (label==\"19\"):\n                return create_model19(embedding_matrix) \n            return None,\"None\"\n            ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a5c80890606763754ed593ab510e072f47741e9"},"cell_type":"markdown","source":"**Funcion to calculate F1 Metric and Plot the Confusion Matrix**"},{"metadata":{"trusted":true,"_uuid":"8e460169d3fd30186ea6c7d5f64d5c0fe2477155","_kg_hide-input":true},"cell_type":"code","source":"from scikitplot.metrics import plot_confusion_matrix\nfrom sklearn.metrics import confusion_matrix\ndef best_F1 (v_y ,t_y,show_plot=False):\n    bs=0\n    bt=0\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        score = metrics.f1_score(v_y, (t_y>=thresh).astype(int))\n        if score >= bs:\n            bt = thresh\n            bs = score  \n    if (show_plot):        \n        print(\"\\n Best F1 score at threshold %2.4f is %2.4f \\n\" % (bt, bs))\n        plot_confusion_matrix(v_y, np.array(pd.Series(t_y.reshape(-1,)).map(lambda x:1 if x>=bt else 0)))\n        d=confusion_matrix(v_y, (t_y>=bt).astype(int))\n        print (\"Total:\",d.sum(),\"/Insincere:\",d[1,:].sum(),\"/Sincere:\",d[0,:].sum())\n        print (\"Total Erros:\",(d[0,1]+d[1,0]),\"/Insincere: %2f\"%((d[0,1]/(d[0,1]+d[1,1]))),\"/Sincere: %2f\" % ((d[1,0]/(d[1,0]+d[0,0]))))\n        \n    return bt, bs","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9382badb2e4de273aeea9a6072db57f1344dfb6f"},"cell_type":"markdown","source":"**F1 Function to use in early stop callback - Evaluate Model 1 Function**"},{"metadata":{"_uuid":"ac90f068745890d9efe6cee7cf9a44f3c00250fb","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from keras import backend as K\nfrom keras.callbacks import Callback\nfrom sklearn.metrics import confusion_matrix, f1_score, precision_score, recall_score\n\ndef f1(y_true, y_pred):\n    def recall(y_true, y_pred):\n        \"\"\"Recall metric.\n\n        Only computes a batch-wise average of recall.\n\n        Computes the recall, a metric for multi-label classification of\n        how many relevant items are selected.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n        recall = true_positives / (possible_positives + K.epsilon())\n        return recall\n\n    def precision(y_true, y_pred):\n        \"\"\"Precision metric.\n\n        Only computes a batch-wise average of precision.\n\n        Computes the precision, a metric for multi-label classification of\n        how many selected items are relevant.\n        \"\"\"\n        true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n        predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n        precision = true_positives / (predicted_positives + K.epsilon())\n        return precision\n    precision = precision(y_true, y_pred)\n    recall = recall(y_true, y_pred)\n    return 2*((precision*recall)/(precision+recall+K.epsilon()))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"865dec499efde9a86d378f4ef3a5e31a1b382f07"},"cell_type":"markdown","source":"'**Function to record results in a pandas dataframe**"},{"metadata":{"trusted":true,"_uuid":"9bb2a5c13181918e2c4ecee4c637b30bfef7e766","_kg_hide-input":true},"cell_type":"code","source":"def rec_results (Model,Embedding,Thresh,Score,StartTime):\n    global resultsdf\n    resultsdf = resultsdf.append({'Model':Model,\n                                  'Method':method,\n                                  'Embedding':Embedding,\n                                  'Pretext':pretext_proc,\n                                  'Weigth':wgt,\n                                  'F1':Score,\n                                  \"Threshold\":Thresh,\n                                  \"Duration\":str(round((time.time()-StartTime)/60,0)),\n                                  'Epochs':epochs,\n                                  'EpochsUsed':epoc_used,\n                                  'Trainable':Trainable,\n                                  'Opt':opt,\n                                  'MaxLength':maxlen,\n                                  'MaxFeatures':max_features,                                  \n                                  'batch_size':batch_size,\n                                  'patience':patience,\n                                  'num_lstm':num_lstm,\n                                  'rate_drop_lstm':rate_drop_lstm,\n                                  'num_dense':num_dense,\n                                  'rate_drop_dense':rate_drop_dense,\n                                  'rate_drop_spatial':rate_drop_spatial,\n                                  'opt':opt,\n                                  'met':met,\n                                  'es_mon':es_mon,\n                                  'es_mode':es_mode,\n                                  'Date':datetime.now().strftime(\"%d-%m-%Y %H:%M\")}, ignore_index=True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"de8e14a924f40fb55dfc7fafc5eeae48861f47e2"},"cell_type":"markdown","source":"**Function to load Embedding files (Glove, Paragram, Wiki and Google)**"},{"metadata":{"trusted":true,"_uuid":"6d0fdb7ceeb9923609ee5c7934fb21373a20cc5c","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def load_embedding (emb,word_index) :\n        estart = time.time()\n        print('Indexing '+emb+' vectors')\n        if (emb==\"Glove\"):\n            EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n            def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n            embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n        if (emb==\"Google\"):\n            EMBEDDING_FILE = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n            embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)\n        if (emb==\"Paragram\"):\n            EMBEDDING_FILE =  '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt' \n            def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n            embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n        if (emb==\"Wiki\"):\n            EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'    \n            def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n            embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n        \n        print(\"Vector\",EMBEDDING_FILE )\n        print(\"End Indexing:\",(str(timedelta(seconds=(time.time()-estart)))) )\n        estart = time.time()\n        print('Preparing embedding matrix')\n        \n        out_of_features=0\n        out_of_embedding=0\n        in_embedding=0\n        if (emb==\"Wiki\" or emb==\"Glove\" or emb==\"Paragram\"):\n            all_embs = np.stack(embeddings_index.values())\n            emb_mean,emb_std = all_embs.mean(), all_embs.std()\n            embed_size = all_embs.shape[1]\n\n            #word_index = tokenizer.word_index\n            nb_words = min(max_features, len(word_index))+1\n            np.random.seed(2018)\n            embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n            for word, i in word_index.items():\n                if i >= max_features: \n                    out_of_features+=1\n                    continue\n                embedding_vector = embeddings_index.get(word)\n                if embedding_vector is not None: \n                    embedding_matrix[i] = embedding_vector\n                    in_embedding+=1\n                else:\n                    out_of_embedding+=1\n        else: #\"Google\"\n            embed_size=300\n            #word_index = tokenizer.word_index\n            nb_words = min(max_features, len(word_index))+1\n            np.random.seed(2018)\n            embedding_matrix= (np.random.rand(nb_words, embed_size) - 0.5) / 5.0\n            #embedding_matrix= (np.random.normal(nb_words, embed_size) - 0.5) / 5.0\n            for word, i in word_index.items():\n                if i >= max_features:\n                    out_of_features+=1\n                    continue\n                if word in embeddings_index:\n                    embedding_vector = embeddings_index.get_vector(word)\n                    embedding_matrix[i] = embedding_vector\n                    in_embedding+=1\n                else:\n                    out_of_embedding+=1\n        print(\"Total Vocabulary:\",len(word_index))\n        print(\"Out of Features:\", out_of_features)\n        print (\"Out of Embeddings:\",out_of_embedding)\n        print (\"Mapped Embeddings:\",in_embedding)\n        print(\"End Preparing embedding matrix:\",(str(timedelta(seconds=(time.time()-estart)))) )\n        del embeddings_index\n        return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8fa6cc943a9be6f71209968d833f945e36111085"},"cell_type":"markdown","source":"**Evaluate Model 0 -Standart fit method**"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"e6a1400890dbc6192ae1bff6c1c6a9401629082c"},"cell_type":"code","source":"def evaluate_model (label,emb, embedding_matrix,train_X,train_y,t_X, t_y, v_X, v_y, epochs, batch_size,patience):\n\n        global resultsdf \n        global pred_val_ytrain_y\n        global prev_test_y\n        global met\n\n        best_thresh = 0.0\n        best_score = 0.0\n        \n        mstart = time.time()\n        model=None\n        model,mdln = create_model(label, embedding_matrix)\n        print (\"\")\n        print (mdln+'-'+emb)\n        #print(modelx.summary())\n        print(\"Model Fitting:\")\n        model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs, validation_data=(v_X, v_y))\n        print(\"Predicting Values:\")\n        pvy = model.predict([v_X], batch_size=1024, verbose=2)\n        best_thresh,best_score=best_F1(v_y,  pvy[:,0] , False)\n        print(\"\\n\")\n        pty = model.predict([test_X], batch_size=1024, verbose=2)\n        rec_results (mdln,emb,best_thresh,best_score,mstart)\n        import gc;           \n        del model\n        gc.collect()\n        time.sleep(10)  \n        return pvy[:,0],pty[:,0],best_score,best_thresh","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22f5ac819c453c007f7daed214031efaf0f9065a"},"cell_type":"markdown","source":"**Evaluate Model 1- Uses the early stop with F1 Metric **"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"9f5b234f2117ed323a4943a1c5bbcd585f0dea57"},"cell_type":"code","source":"def evaluate_model1 (label,emb, embedding_matrix,train_X,train_y,t_X,t_y,v_X,v_y, epochs, batch_size, patience=0):\n\n        global resultsdf \n        global pred_val_y\n        global prev_test_y\n        global met \n        global epoc_used\n        best_thresh = 0.0\n        best_score = 0.0\n        mstart = time.time()\n\n        \n        model=None\n        model,mdln = create_model(label,embedding_matrix)\n        print (\"\")\n        print (mdln+'-'+emb)\n\n        model_checkpoint = ModelCheckpoint('./model.hdf5', monitor=es_mon, mode=es_mode,\n                                      verbose=True, save_best_only=True, \n                                      save_weights_only=False)\n    \n        early_stopping = EarlyStopping(monitor=es_mon, mode=es_mode, patience=patience,verbose=True)\n        callbacks = [ early_stopping, model_checkpoint]\n        print(\"Model Fitting:\")\n        hist= model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs,\n              shuffle=True, verbose=True, validation_data=(v_X, v_y),\n              callbacks=callbacks)\n        #print(\"History\\n\",hist.history.keys()) \n        #print(\"F1\",hist.history['f1'])\n        epoc_used=len(hist.history[es_mon])\n        ### Getting the Best Model\n        model.load_weights('./model.hdf5')   \n\n        print(\"Predicting Values:\")\n        #predictions_valid = bestmodel.predict(X_valid.astype('float32'), batch_size=batch_size, verbose=2)\n        pvy = model.predict([v_X], batch_size=batch_size, verbose=2)\n        best_thresh,best_score=best_F1(v_y,pvy[:,0] , False)             \n        pty = model.predict([test_X], batch_size=1024, verbose=2)\n        rec_results (mdln,emb,best_thresh,best_score,mstart)\n        import gc;           \n        del model\n        gc.collect()\n        time.sleep(10)\n        return  pvy[:,0],pty[:,0],best_score,best_thresh","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1a483c373b13ce3478641ec1441397dd917085e"},"cell_type":"markdown","source":"**Evaluate Model 2- Manualy generates epochs and uses F1 metric to early stop **"},{"metadata":{"trusted":true,"_uuid":"47d3334b33931ed9e176d52b462fb1858eb97a24","_kg_hide-input":true},"cell_type":"code","source":"def evaluate_model2 (label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):\n    \n    global resultsdf \n    global pred_val_y\n    global prev_test_y\n    global met\n\n    best_thresh = 0.0\n    best_score = 0.0\n\n    mstart = time.time()\n    model=None\n    model,mdln = create_model(label, embedding_matrix)\n    print (\"\")\n    print (mdln+'-'+emb)\n    bs=0\n    pt=0\n    print(\"Model Fitting:\")\n    for e in range(epochs):\n        model.fit(t_X, t_y, batch_size=batch_size, epochs=1, validation_data=(v_X, v_y))\n        print(\"Predicting Values:\")\n        pvy = model.predict([v_X], batch_size=1024, verbose=0)\n        best_thresh,best_score=best_F1(v_y,pvy[:,0], False )\n        print(\"epoch \",e+1,\"/\",epochs)\n        if (best_score > bs):          \n            print(\"F1 Score Improved from %2.4f to %2.4f\" % (bs,best_score))\n            bs = best_score\n            bt = best_thresh\n            epoc_used=e+1\n            model.save_weights('./model.hdf5') \n        else:\n            print(\"F1 Score not improved\")\n            pt=pt+1\n            if (pt>patience):\n                break\n    model.load_weights('./model.hdf5')             \n    pty = model.predict([test_X], batch_size=1024, verbose=0)\n    rec_results (mdln,emb,bt,bs,mstart) \n    import gc;           \n    del model\n    gc.collect()\n    time.sleep(10)\n    return pvy[:,0],pty[:,0],bs,bt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"95eb066e7d6739d279fda9d5607ed3fbc5ea0f1c"},"cell_type":"markdown","source":"**Evaluate Model 3 -Using CLR method**"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true,"_uuid":"580ec0c8272895b80450e75180f25eb0acb0fc41"},"cell_type":"code","source":"def evaluate_model3 (label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):\n    \n    global resultsdf \n    global pred_val_y\n    global prev_test_y\n    global met\n    global clr\n    global epoc_used \n    clr = CyclicLR(base_lr=0.001, max_lr=0.01,\n               step_size=300., mode='exp_range',\n               gamma=0.99994)\n    callback = [clr,]\n\n    best_thresh = 0.0\n    best_score = 0.0\n\n    mstart = time.time()\n    model=None\n    model,mdln = create_model(label, embedding_matrix)\n    print (\"\")\n    print (mdln+'-'+emb)\n    bs=0\n    pt=0\n    print(\"Model Fitting:\")\n    for e in range(epochs):\n        model.fit(t_X, t_y, batch_size=batch_size, epochs=1, validation_data=(v_X, v_y),callbacks = callback,verbose=2)\n        print(\"Predicting Values:\")\n        pvy = model.predict([v_X], batch_size=1024, verbose=0)\n        print(\"F1 - Score:\")\n        best_thresh,best_score=best_F1(v_y,pvy[:,0] )\n        print(\"epoch \",e+1,\"/\",epochs)\n        if (best_score > bs):          \n            print(\"F1 Score Improved from %2.4f to %2.4f\" % (bs,best_score))\n            bs = best_score\n            bt = best_thresh\n            epoc_used=e+1\n            model.save_weights('./model.hdf5') \n        else:\n            print(\"F1 Score not improved\")\n            pt=pt+1\n            if (pt>patience):\n                break\n    model.load_weights('./model.hdf5')             \n    pty = model.predict([test_X], batch_size=1024, verbose=0)\n    rec_results (mdln,emb,bt,bs,mstart) \n    import gc;           \n    del model\n    gc.collect()\n    time.sleep(10)\n    return pvy[:,0],pty[:,0],bs,bt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d415880899ccfd65ed261033b76961153d9dacfc"},"cell_type":"markdown","source":"**Evaluate Model 4 - Method EarlyStop + CLR**"},{"metadata":{"_uuid":"2df0586fbf1c14c9aef4238d941f32448b633657","_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"def evaluate_model4 (label,emb, embedding_matrix,train_X,train_y,t_X,t_y,v_X,v_y, epochs, batch_size, patience=0):\n\n        global resultsdf \n        global pred_val_y\n        global prev_test_y\n        global met \n        global clr\n        global epoc_used \n        clr = CyclicLR(base_lr=0.001, max_lr=0.01,\n                   step_size=300., mode='exp_range',\n                   gamma=0.99994)\n        best_thresh = 0.0\n        best_score = 0.0\n        \n        mstart = time.time()\n        model=None\n        model,mdln = create_model(label,embedding_matrix)\n        print (\"\")\n        print (mdln+'-'+emb)\n        model_checkpoint = ModelCheckpoint('./model.hdf5', monitor=es_mon, mode=es_mode,\n                                      verbose=True, save_best_only=True, \n                                      save_weights_only=False)\n    \n        early_stopping = EarlyStopping(monitor=es_mon, mode=es_mode, patience=patience,verbose=True)\n        callbacks = [ early_stopping, model_checkpoint,clr,]\n        print(\"Model Fitting:\")\n        hist= model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs,\n              shuffle=True, verbose=True, validation_data=(v_X, v_y),\n              callbacks=callbacks)\n        epoc_used=len(hist.history[es_mon])  \n        ### Getting the Best Model\n        print (\"Getting the Best Model\")  \n        model.load_weights('./model.hdf5')   \n\n        print(\"Predicting Values:\")\n        #predictions_valid = bestmodel.predict(X_valid.astype('float32'), batch_size=batch_size, verbose=2)\n        pvy = model.predict([v_X], batch_size=batch_size, verbose=2)\n        best_thresh,best_score=best_F1(v_y,pvy[:,0], False )             \n        pty = model.predict([test_X], batch_size=1024, verbose=2)\n        rec_results (mdln,emb,best_thresh,best_score,mstart)\n        import gc;           \n        del model\n        gc.collect()\n        time.sleep(10)\n        return  pvy[:,0],pty[:,0],best_score,best_thresh","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4c0509805c4c079febff19a6e6daf5c87577b779"},"cell_type":"markdown","source":"**Evaluate Model 5 - Manual Epochs + CLR**"},{"metadata":{"trusted":true,"_uuid":"a93c59d4333983f556ae6574142c08b72c0ab6c9","_kg_hide-input":true},"cell_type":"code","source":"def evaluate_model5 (label,emb, embedding_matrix,train_X,train_y, t_X, t_y, v_X, v_y, epochs, batch_size, patience=0):\n    \n    global resultsdf \n    global pred_val_y\n    global prev_test_y\n    global met\n    global clr\n    global epoc_used \n    clr = CyclicLR(base_lr=0.001, max_lr=0.01,\n               step_size=300., mode='exp_range',\n               gamma=0.99994)\n    callback = [clr,]\n    best_thresh = 0.0\n    best_score = 0.0\n\n    mstart = time.time()\n    model=None\n    model,mdln = create_model(label, embedding_matrix)\n    print (\"\")\n    print (mdln+'-'+emb)\n    bs=0\n    pt=0\n    print(\"Model Fitting:\")\n    for e in range(epochs):\n        model.fit(t_X, t_y, batch_size=batch_size, epochs=1, validation_data=(v_X, v_y),verbose=2)\n        print(\"Predicting Values:\")\n        pvy = model.predict([v_X], batch_size=1024, verbose=0)\n        best_thresh = 0.0\n        best_score = 0.0\n        for thresh in np.arange(0.1, 0.501, 0.01):\n            thresh = np.round(thresh, 2)\n            score = metrics.f1_score(v_y, (pvy[:,0]  > thresh).astype(int))\n            if score > best_score:\n                print(\"F1COnf Score Improved from %2.4f to %2.4f\" % (best_score,score))\n                best_thresh = thresh\n                best_score = score\n        best_thresh = 0.0\n        best_score = 0.0\n        print(\"F1 - Score:\")\n        best_thresh,best_score=best_F1(v_y,pvy[:,0] )\n        print(\"epoch \",e+1,\"/\",epochs)\n        epoc_used=e+1\n        if (best_score > bs):          \n            print(\"F1 Score Improved from %2.4f to %2.4f\" % (bs,best_score))\n            bs = best_score\n            bt = best_thresh\n            \n    pty = model.predict([test_X], batch_size=1024, verbose=0)\n    rec_results (mdln,emb,bt,bs,mstart) \n    import gc;           \n    del model\n    gc.collect()\n    time.sleep(10)\n    return pvy[:,0],pty[:,0],bs,bt","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ec89b11f2906e0c924378ee99c60684c8b056a4"},"cell_type":"markdown","source":"**Evaluate Model 6 - Early Stop , Reduce LR **"},{"metadata":{"trusted":true,"_uuid":"f6a6337be245f9e099e384c832282d8570f99bf7"},"cell_type":"code","source":"from keras.callbacks import *\ndef evaluate_model6 (label,emb, embedding_matrix,train_X,train_y,t_X,t_y,v_X,v_y, epochs, batch_size, patience=0):\n\n        global resultsdf \n        global pred_val_y\n        global prev_test_y\n        global met \n        global epoc_used\n        best_thresh = 0.0\n        best_score = 0.0\n        mstart = time.time()\n\n        \n        model=None\n        model,mdln = create_model(label,embedding_matrix)\n        print (\"\")\n        print (mdln+'-'+emb)\n        \n        model_checkpoint = ModelCheckpoint('./model.hdf5', monitor=es_mon, mode=es_mode,\n                                      verbose=True, save_best_only=True, \n                                      save_weights_only=False)\n        reduce_lr = ReduceLROnPlateau(monitor=es_mon, factor=0.5, patience=1, min_lr=0.0001, verbose=True)\n        early_stopping = EarlyStopping(monitor=es_mon,min_delta=0.0001, mode='auto', patience=patience,verbose=True)\n        \n        #callbacks = [ early_stopping, model_checkpoint]\n        callbacks = [ early_stopping,model_checkpoint,reduce_lr]    \n    \n        print(\"Model Fitting:\")\n        hist= model.fit(t_X, t_y, batch_size=batch_size, epochs=epochs,\n              shuffle=True, verbose=True, validation_data=(v_X, v_y),\n              callbacks=callbacks)\n        #print(\"History\\n\",hist.history.keys()) \n        #print(\"F1\",hist.history['f1'])\n        epoc_used=len(hist.history[es_mon])\n        ### Getting the Best Model\n        model.load_weights('./model.hdf5')   \n\n        print(\"Predicting Values:\")\n        #predictions_valid = bestmodel.predict(X_valid.astype('float32'), batch_size=batch_size, verbose=2)\n        pvy = model.predict([v_X], batch_size=batch_size, verbose=2)\n        best_thresh,best_score=best_F1(v_y,pvy[:,0] , False)             \n        pty = model.predict([test_X], batch_size=1024, verbose=2)\n        rec_results (mdln,emb,best_thresh,best_score,mstart)\n        import gc;           \n        del model\n        gc.collect()\n        time.sleep(10)\n        return  pvy[:,0],pty[:,0],best_score,best_thresh","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f6bba09b1a7d343dd5bda6cc8043d5922db9e17"},"cell_type":"markdown","source":"**Evaluate Model -Strafied K-Fold**"},{"metadata":{"trusted":true,"_uuid":"f1923fee3a5bbf2b20aa7fa6e7b833b2836deec0","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"def evaluate_stratifiedkfold (label,mt,emb,train_X,train_y, test_X, n_splits, stop_split, epochs, batch_size,patience):\n    \n    global resultsdf \n    global allpred_val_y\n    global allprev_test_y\n    global mn\n    global method\n    method=mt\n    random_seed = 2018\n    train_stratified=np.zeros([len(train_X),(stop_split)])\n    test_stratified = np.zeros([len(test_X),(stop_split)])\n    msstart = time.time()\n    embedding_matrix=embedding_matrix_combined\n    if  (emb==\"Glove\"):\n        embedding_matrix=embedding_matrix_1\n    if  (emb==\"Wiki\"):\n        embedding_matrix=embedding_matrix_2\n    if  (emb==\"Paragram\"):\n        embedding_matrix=embedding_matrix_3\n    if  (emb==\"Google\"):\n        embedding_matrix=embedding_matrix_4\n    if  (emb==\"Concatenated\"):\n        embedding_matrix=embedding_matrix_concatenated \n\n    splits = list(StratifiedKFold(n_splits=n_splits, shuffle=True, random_state=random_seed).split(train_X, train_y))\n    mn=mn+1\n    bs=np.zeros([stop_split])\n    bt=np.zeros([stop_split])\n    for idx, (train_idx, valid_idx) in enumerate(splits):\n            K.clear_session()\n            X_split = train_X[train_idx]\n            y_split = train_y[train_idx]\n            print (\"Split:\",idx)\n            print (\"Total X-train:\",len(y_split),\" Insincere:\",y_split.sum(),y_split.sum()/ len(y_split) )\n            X_val = train_X[valid_idx]\n            y_val = train_y[valid_idx]\n            print (\"Total X-val:\",len(y_val),\" Insincere:\",y_val.sum(), y_val.sum()/len(y_val))\n            if (mt==\"0\"):\n                pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val, epochs, batch_size ,patience)\n            if (mt==\"1\"):\n                pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model1(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience)\n            if (mt==\"2\"):\n                pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model2(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience) \n            if (mt==\"3\"):\n                pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model3(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience) \n            if (mt==\"4\"):\n                pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model4(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience) \n            if (mt==\"5\"):\n                pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model5(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience) \n            if (mt==\"6\"):\n                pred_train_y, pred_test_y, best_score, best_thresh=evaluate_model6(md,emb,embedding_matrix,train_X,train_y, X_split, y_split, X_val, y_val,epochs, batch_size ,patience) \n            if (best_score > bs[idx]):          \n                print(\"F1 Score Improved from %2.4f to %2.4f\" % (bs[idx],best_score))\n                bs[idx] = best_score\n                bt[idx] = best_thresh\n            #train_stratified[:,idx] =  pred_train_y\n            test_stratified[:,idx] = pred_test_y\n            if ((idx+1)==stop_split):\n                break\n    \n    train_pred =  train_stratified.mean(axis=1)\n    test_pred  = test_stratified.mean(axis=1)\n    #train_pred =  train_stratified.max(axis=1)\n    #test_pred  = test_stratified.max(axis=1)\n    #print(\"F1 - Score - K-Fold:\")\n    #best_thresh,best_score=best_F1(train_y,train_pred,False )\n    if (stop_split > 1):\n        rec_results (\"Model Mean:\"+label+\"-skfold- \"+str(n_splits)+\"Stop: \"+str(stop_split),emb,bt.min(),bs.mean(),msstart)\n    \n    return train_pred,test_pred,bs.max(),bt.min()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"330401c9c33e353aca71cc6b86d3f34d5050ec9b"},"cell_type":"markdown","source":"**Load Train,Validation and Test Data. \nWith or without pre-preprocessing.**"},{"metadata":{"trusted":true,"_uuid":"8fdca1e249fdcaf89fdd8d4f07d3f3252435bdec","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"###\n###  Load Train,Val and Test \n###\n\ntrain_X, test_X, train_y, test_df, word_index = load_and_prec(True)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aebe46c0a10c2c9f76f5a8fb98551435a6099a4e"},"cell_type":"markdown","source":"**Load Embedding Matrix from Glove,Wki,Paragram and Google. **\nCreate Combination of Matrix"},{"metadata":{"trusted":true,"_uuid":"ff7b8b27a1f97cbdff16476c0c99712ac6d9efc4","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"###\n### Load Embedding Matrix\n###\nembedding_matrix_1 = load_embedding(\"Glove\",word_index)\n#embedding_matrix_2 = load_embedding(\"Wiki\",word_index)\nembedding_matrix_3 = load_embedding(\"Paragram\",word_index)\n#embedding_matrix_4 = load_embedding(\"Google\",word_index)\n## Simple average: http://aclweb.org/anthology/N18-2031\n\n# We have presented an argument for averaging as\n# a valid meta-embedding technique, and found experimental\n# performance to be close to, or in some cases \n# better than that of concatenation, with the\n# additional benefit of reduced dimensionality  \n\n\n## Unweighted DME in https://arxiv.org/pdf/1804.07983.pdf\n\n# “The downside of concatenating embeddings and \n#  giving that as input to an RNN encoder, however,\n#  is that the network then quickly becomes inefficient\n#  as we combine more and more embeddings.”\n  \n# embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_2, embedding_matrix_3], axis = 0)\nembedding_matrix_combined = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)\nnp.shape(embedding_matrix_combined)\n\nembedding_matrix_concatenated = np.concatenate((embedding_matrix_1, embedding_matrix_3), axis=1)  \n#del embedding_matrix_1, embedding_matrix_2, embedding_matrix_3, embedding_matrix_4\n#gc.collect()\nnp.shape(embedding_matrix_concatenated)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"3374663b48aee4450c1602fce1f460ed451cb91c"},"cell_type":"markdown","source":"**Run Models\n**"},{"metadata":{"trusted":true,"_uuid":"16dffb7853c3e94281d5a9e38bff5494bd322e49","_kg_hide-input":true,"_kg_hide-output":true,"scrolled":true},"cell_type":"code","source":"###\n### Run All Models\n###\nset_all_parameters()\n\nresultsdf = pd.DataFrame(columns=['Model','F1','Embedding','Pretext','Weigth','Duration','Method',\n                                  'Epochs','EpochsUsed','patience','batch_size'\n                                  ,'MaxLength','MaxFeatures' ])\n\n\n### Matrix of Predictions - Validation and Test\n### Stores All Predicted Values\n\nallpred_train_y=np.zeros([len(train_X),(len(model_list))])\nallpred_test_y=np.zeros([len(test_X),(len(model_list))])\nallbest=np.zeros([(len(model_list))])\nallthresh=np.zeros([(len(model_list))])\n\nepoc_used=0\nmn=-1 \n\n#model_list.append([\"18\",1,\"5\",\"Combine\",8,pat,bs,splits,stop,nh,nd,ndh,ndd,nds,loss_bin,opt_rmsprop,met_acc,mon_acc,mode_acc])\nfor ne,item in  enumerate(model_list): \n    md=item[0]  # Model to Run\n    wgt=item[1]    # Model weigth for essembling\n    mt=item[2]   # Model Evaluation Method\n    emb=item[3] # Embeddings\n    epochs=item[4] # Number of Epochs\n    patience=item[5] # Patience - Wait patience epochs if score does not improve\n    batch_size=item[6] # Batch Size to Fit Model\n    nsplits=item[7]   # Number of Splits for Stratified k-fold\n    stop_split=item[8]    # Number of Splits to run in Stratified k-fold\n    num_lstm=item[9]  # Size of hidden layers in models (lstm and gru)\n    num_dense=item[10] # Size of hidden layers in model - dense\n    rate_drop_lstm = item[11] # lstm and gru - drop-out rates\n    rate_drop_dense = item[12] # dense drop-out rates\n    rate_drop_spatial=item[13]  # spatial drop-out rate\n    loss=item[14] # Model Loss\n    opt=item[15]  #Models Optmizer\n    met=item[16]  #Models Metric\n    es_mon=item[17] # Early stop monitor \n    es_mode=item[18]  # Early stop mode\n    print (mt)\n    print (\"md-\"+md)\n    trainp,testp,bs,bt=evaluate_stratifiedkfold (md,mt,emb,train_X,train_y, test_X, nsplits, stop_split, epochs, batch_size,patience)\n    allbest[mn]=bs\n    allthresh[mn]=bt\n    #allpred_train_y[:,mn] = trainp\n    allpred_test_y[:,mn] = testp\n    print (\"FIM-\"+md)\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad19564cfeb45f4bfcf520af253ae239bdd82735"},"cell_type":"markdown","source":"**Calculate weights and apply to results**"},{"metadata":{"trusted":true,"_uuid":"22cd300ddf59e25266a21bddbc752b0908331302","_kg_hide-input":true,"scrolled":false},"cell_type":"code","source":"#### Aplicando Pesos nos Modelos\n#### Aplicando Pesos nos Modelos\ntwgt=0\nwgt_test_y=0\nwgt_train_y=0\nessemblet=\"Essemble: \"\nfor ne,item in  enumerate(model_list):\n    wg=item[1]\n    twgt=twgt+wg\n    #wgt_train_y=wgt_train_y+(allpred_train_y[:,ne]*wg)\n    wgt_test_y =wgt_test_y +(allpred_test_y[:,ne]*wg)\n    essemblet=essemblet+item[0]+\" \"\n#wgt_train_y=wgt_train_y/twgt\nwgt_test_y=wgt_test_y/twgt\n\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e735e916b119a4148aaec87772f76130a000310f"},"cell_type":"markdown","source":"**Write Output to submission file**"},{"metadata":{"trusted":true,"_uuid":"ed7c755da2a2f96f4f41d58722c4d1f1af78a168","_kg_hide-input":true},"cell_type":"code","source":"#best_thresh,best_score=best_F1(train_y,wgt_train_y,True)\n#\nprint (allthresh)\nprint (allthresh.min())\nprint (allthresh.mean())\n\n# Write the output\ny_te = (np.array(wgt_test_y) >= allthresh.min()).astype(np.int)\nsubmit_df = pd.DataFrame({\"qid\": test_df[\"qid\"], \"prediction\": y_te})\nsubmit_df.to_csv(\"./submission.csv\",index=False)\n#display(submit_df)\nrec_results ('Time Elapsed','',0,0,bstart)\ndisplay (resultsdf)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"994c4c3cd7ddeadf3718933bd74edd5700ddb288"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}