{"cells":[{"metadata":{"_uuid":"a98ef934cae654ed9818e0a34e1bdb570de19388"},"cell_type":"markdown","source":"some reference:\nhttps://www.kaggle.com/gmhost/gru-capsule by Puck Wang\n\nhttps://www.kaggle.com/shujian/single-rnn-with-4-folds-clr by shujian\n\nhttps://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings by SRK\n\nhttps://www.kaggle.com/christofhenkel/how-to-preprocessing-when-using-embeddings by Dieter\n\nhttps://www.kaggle.com/shujian/mix-of-nn-models-based-on-meta-embedding by shujian\n\nTell me if missed any"},{"metadata":{"trusted":true,"_uuid":"b378958a9606ac48fe0dc54e24bed4cd503e0ac7"},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nfrom tqdm import tqdm\ntqdm.pandas()\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras.initializers import *\nfrom keras.optimizers import *\nimport keras.backend as K\nfrom keras.callbacks import *\nimport tensorflow as tf\nimport os\nimport time\nimport gc\nimport re\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/train.csv\")\ntest = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \", train.shape)\nprint(\"Test shape : \", test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d336bfef799c16f12f2d02ffa4f3c2eaaf6ef34"},"cell_type":"code","source":"train[\"question_text\"] = train[\"question_text\"].str.lower()\ntest[\"question_text\"] = test[\"question_text\"].str.lower()\n\npuncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', \n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', \n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', \n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', \n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\ndef clean_text(x):\n\n    x = str(x)\n    for punct in puncts:\n        x = x.replace(punct, f' {punct} ')\n    return x\n\n\ntrain[\"question_text\"] = train[\"question_text\"].apply(lambda x: clean_text(x))\ntest[\"question_text\"] = test[\"question_text\"].apply(lambda x: clean_text(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"87b0cf1880df72c47d8a882e6441aaa07dacfd9c"},"cell_type":"code","source":"## some config values \nembed_size = 300 # how big is each word vector\nmax_features = None # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 72 # max number of words in a question to use #99.99%\n\n## fill up the missing values\nX = train[\"question_text\"].fillna(\"_na_\").values\nX_test = test[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(X))\n\nX = tokenizer.texts_to_sequences(X)\nX_test = tokenizer.texts_to_sequences(X_test)\n\n## Pad the sentences \nX = pad_sequences(X, maxlen=maxlen)\nX_test = pad_sequences(X_test, maxlen=maxlen)\n\n## Get the target values\nY = train['target'].values\n\nsub = test[['qid']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"259d359c2fd45efd7ccd1a18db69fbb7fe7ad8d2"},"cell_type":"code","source":"del train, test\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"26e110fe2c74167bf02fda50e7cd9cbb897dca57"},"cell_type":"code","source":"word_index = tokenizer.word_index\nmax_features = len(word_index)+1\ndef load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if o.split(\" \")[0] in word_index)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100 and o.split(\" \")[0] in word_index )\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100 and o.split(\" \")[0] in word_index)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n    \n    embedding_matrix = np.random.normal(emb_mean, emb_std, (max_features, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2505794d8ebc30884ccc6a9a9a17a3a1af12bd29"},"cell_type":"code","source":"embedding_matrix_1 = load_glove(word_index)\n#embedding_matrix_2 = load_fasttext(word_index)\nembedding_matrix_3 = load_para(word_index)\nembedding_matrix = np.mean((embedding_matrix_1, embedding_matrix_3), axis=0)  \ndel embedding_matrix_1, embedding_matrix_3\ngc.collect()\nnp.shape(embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b75ab7890f8d51c008b47d994011f88a35de36a7","trusted":true},"cell_type":"code","source":"from keras import backend as K\nfrom keras import activations\nfrom keras import initializers\nfrom keras import regularizers\nfrom keras import constraints\nfrom keras.engine import Layer\nfrom keras.engine import InputSpec\nfrom keras.objectives import categorical_crossentropy\nfrom keras.objectives import sparse_categorical_crossentropy\n\n\nclass CRF(Layer):\n    \"\"\"An implementation of linear chain conditional random field (CRF).\n    An linear chain CRF is defined to maximize the following likelihood function:\n    $$ L(W, U, b; y_1, ..., y_n) := \\frac{1}{Z} \\sum_{y_1, ..., y_n} \\exp(-a_1' y_1 - a_n' y_n\n        - \\sum_{k=1^n}((f(x_k' W + b) y_k) + y_1' U y_2)), $$\n    where:\n        $Z$: normalization constant\n        $x_k, y_k$:  inputs and outputs\n    This implementation has two modes for optimization:\n    1. (`join mode`) optimized by maximizing join likelihood, which is optimal in theory of statistics.\n       Note that in this case, CRF must be the output/last layer.\n    2. (`marginal mode`) return marginal probabilities on each time step and optimized via composition\n       likelihood (product of marginal likelihood), i.e., using `categorical_crossentropy` loss.\n       Note that in this case, CRF can be either the last layer or an intermediate layer (though not explored).\n    For prediction (test phrase), one can choose either Viterbi best path (class indices) or marginal\n    probabilities if probabilities are needed. However, if one chooses *join mode* for training,\n    Viterbi output is typically better than marginal output, but the marginal output will still perform\n    reasonably close, while if *marginal mode* is used for training, marginal output usually performs\n    much better. The default behavior is set according to this observation.\n    In addition, this implementation supports masking and accepts either onehot or sparse target.\n    # Examples\n    ```python\n        model = Sequential()\n        model.add(Embedding(3001, 300, mask_zero=True)(X)\n        # use learn_mode = 'join', test_mode = 'viterbi', sparse_target = True (label indice output)\n        crf = CRF(10, sparse_target=True)\n        model.add(crf)\n        # crf.accuracy is default to Viterbi acc if using join-mode (default).\n        # One can add crf.marginal_acc if interested, but may slow down learning\n        model.compile('adam', loss=crf.loss_function, metrics=[crf.accuracy])\n        # y must be label indices (with shape 1 at dim 3) here, since `sparse_target=True`\n        model.fit(x, y)\n        # prediction give onehot representation of Viterbi best path\n        y_hat = model.predict(x_test)\n    ```\n    # Arguments\n        units: Positive integer, dimensionality of the output space.\n        learn_mode: Either 'join' or 'marginal'.\n            The former train the model by maximizing join likelihood while the latter\n            maximize the product of marginal likelihood over all time steps.\n        test_mode: Either 'viterbi' or 'marginal'.\n            The former is recommended and as default when `learn_mode = 'join'` and\n            gives one-hot representation of the best path at test (prediction) time,\n            while the latter is recommended and chosen as default when `learn_mode = 'marginal'`,\n            which produces marginal probabilities for each time step.\n        sparse_target: Boolean (default False) indicating if provided labels are one-hot or\n            indices (with shape 1 at dim 3).\n        use_boundary: Boolean (default True) indicating if trainable start-end chain energies\n            should be added to model.\n        use_bias: Boolean, whether the layer uses a bias vector.\n        kernel_initializer: Initializer for the `kernel` weights matrix,\n            used for the linear transformation of the inputs.\n            (see [initializers](../initializers.md)).\n        chain_initializer: Initializer for the `chain_kernel` weights matrix,\n            used for the CRF chain energy.\n            (see [initializers](../initializers.md)).\n        boundary_initializer: Initializer for the `left_boundary`, 'right_boundary' weights vectors,\n            used for the start/left and end/right boundary energy.\n            (see [initializers](../initializers.md)).\n        bias_initializer: Initializer for the bias vector\n            (see [initializers](../initializers.md)).\n        activation: Activation function to use\n            (see [activations](../activations.md)).\n            If you pass None, no activation is applied\n            (ie. \"linear\" activation: `a(x) = x`).\n        kernel_regularizer: Regularizer function applied to\n            the `kernel` weights matrix\n            (see [regularizer](../regularizers.md)).\n        chain_regularizer: Regularizer function applied to\n            the `chain_kernel` weights matrix\n            (see [regularizer](../regularizers.md)).\n        boundary_regularizer: Regularizer function applied to\n            the 'left_boundary', 'right_boundary' weight vectors\n            (see [regularizer](../regularizers.md)).\n        bias_regularizer: Regularizer function applied to the bias vector\n            (see [regularizer](../regularizers.md)).\n        kernel_constraint: Constraint function applied to\n            the `kernel` weights matrix\n            (see [constraints](../constraints.md)).\n        chain_constraint: Constraint function applied to\n            the `chain_kernel` weights matrix\n            (see [constraints](../constraints.md)).\n        boundary_constraint: Constraint function applied to\n            the `left_boundary`, `right_boundary` weights vectors\n            (see [constraints](../constraints.md)).\n        bias_constraint: Constraint function applied to the bias vector\n            (see [constraints](../constraints.md)).\n        input_dim: dimensionality of the input (integer).\n            This argument (or alternatively, the keyword argument `input_shape`)\n            is required when using this layer as the first layer in a model.\n        unroll: Boolean (default False). If True, the network will be unrolled, else a symbolic loop will be used.\n            Unrolling can speed-up a RNN, although it tends to be more memory-intensive.\n            Unrolling is only suitable for short sequences.\n    # Input shape\n        3D tensor with shape `(nb_samples, timesteps, input_dim)`.\n    # Output shape\n        3D tensor with shape `(nb_samples, timesteps, units)`.\n    # Masking\n        This layer supports masking for input data with a variable number\n        of timesteps. To introduce masks to your data,\n        use an [Embedding](embeddings.md) layer with the `mask_zero` parameter\n        set to `True`.\n    \"\"\"\n\n    def __init__(self, units,\n                 learn_mode='join',\n                 test_mode=None,\n                 sparse_target=False,\n                 use_boundary=True,\n                 use_bias=True,\n                 activation='linear',\n                 kernel_initializer='glorot_uniform',\n                 chain_initializer='orthogonal',\n                 bias_initializer='zeros',\n                 boundary_initializer='zeros',\n                 kernel_regularizer=None,\n                 chain_regularizer=None,\n                 boundary_regularizer=None,\n                 bias_regularizer=None,\n                 kernel_constraint=None,\n                 chain_constraint=None,\n                 boundary_constraint=None,\n                 bias_constraint=None,\n                 input_dim=None,\n                 unroll=False,\n                 **kwargs):\n        super(CRF, self).__init__(**kwargs)\n        self.supports_masking = True\n        self.units = units\n        self.learn_mode = learn_mode\n        assert self.learn_mode in ['join', 'marginal']\n        self.test_mode = test_mode\n        if self.test_mode is None:\n            self.test_mode = 'viterbi' if self.learn_mode == 'join' else 'marginal'\n        else:\n            assert self.test_mode in ['viterbi', 'marginal']\n        self.sparse_target = sparse_target\n        self.use_boundary = use_boundary\n        self.use_bias = use_bias\n\n        self.activation = activations.get(activation)\n\n        self.kernel_initializer = initializers.get(kernel_initializer)\n        self.chain_initializer = initializers.get(chain_initializer)\n        self.boundary_initializer = initializers.get(boundary_initializer)\n        self.bias_initializer = initializers.get(bias_initializer)\n\n        self.kernel_regularizer = regularizers.get(kernel_regularizer)\n        self.chain_regularizer = regularizers.get(chain_regularizer)\n        self.boundary_regularizer = regularizers.get(boundary_regularizer)\n        self.bias_regularizer = regularizers.get(bias_regularizer)\n\n        self.kernel_constraint = constraints.get(kernel_constraint)\n        self.chain_constraint = constraints.get(chain_constraint)\n        self.boundary_constraint = constraints.get(boundary_constraint)\n        self.bias_constraint = constraints.get(bias_constraint)\n\n        self.unroll = unroll\n\n    def build(self, input_shape):\n        self.input_spec = [InputSpec(shape=input_shape)]\n        self.input_dim = input_shape[-1]\n\n        self.kernel = self.add_weight((self.input_dim, self.units),\n                                      name='kernel',\n                                      initializer=self.kernel_initializer,\n                                      regularizer=self.kernel_regularizer,\n                                      constraint=self.kernel_constraint)\n        self.chain_kernel = self.add_weight((self.units, self.units),\n                                            name='chain_kernel',\n                                            initializer=self.chain_initializer,\n                                            regularizer=self.chain_regularizer,\n                                            constraint=self.chain_constraint)\n        if self.use_bias:\n            self.bias = self.add_weight((self.units,),\n                                        name='bias',\n                                        initializer=self.bias_initializer,\n                                        regularizer=self.bias_regularizer,\n                                        constraint=self.bias_constraint)\n        else:\n            self.bias = None\n\n        if self.use_boundary:\n            self.left_boundary = self.add_weight((self.units,),\n                                                 name='left_boundary',\n                                                 initializer=self.boundary_initializer,\n                                                 regularizer=self.boundary_regularizer,\n                                                 constraint=self.boundary_constraint)\n            self.right_boundary = self.add_weight((self.units,),\n                                                  name='right_boundary',\n                                                  initializer=self.boundary_initializer,\n                                                  regularizer=self.boundary_regularizer,\n                                                  constraint=self.boundary_constraint)\n        self.built = True\n\n    def call(self, X, mask=None):\n        if mask is not None:\n            assert K.ndim(mask) == 2, 'Input mask to CRF must have dim 2 if not None'\n\n        if self.test_mode == 'viterbi':\n            test_output = self.viterbi_decoding(X, mask)\n        else:\n            test_output = self.get_marginal_prob(X, mask)\n\n        self.uses_learning_phase = True\n        if self.learn_mode == 'join':\n            train_output = K.zeros_like(K.dot(X, self.kernel))\n            out = K.in_train_phase(train_output, test_output)\n        else:\n            if self.test_mode == 'viterbi':\n                train_output = self.get_marginal_prob(X, mask)\n                out = K.in_train_phase(train_output, test_output)\n            else:\n                out = test_output\n        return out\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[:2] + (self.units,)\n\n    def compute_mask(self, input, mask=None):\n        if mask is not None and self.learn_mode == 'join':\n            return K.any(mask, axis=1)\n        return mask\n\n    def get_config(self):\n        config = {'units': self.units,\n                  'learn_mode': self.learn_mode,\n                  'test_mode': self.test_mode,\n                  'use_boundary': self.use_boundary,\n                  'use_bias': self.use_bias,\n                  'sparse_target': self.sparse_target,\n                  'kernel_initializer': initializers.serialize(self.kernel_initializer),\n                  'chain_initializer': initializers.serialize(self.chain_initializer),\n                  'boundary_initializer': initializers.serialize(self.boundary_initializer),\n                  'bias_initializer': initializers.serialize(self.bias_initializer),\n                  'activation': activations.serialize(self.activation),\n                  'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),\n                  'chain_regularizer': regularizers.serialize(self.chain_regularizer),\n                  'boundary_regularizer': regularizers.serialize(self.boundary_regularizer),\n                  'bias_regularizer': regularizers.serialize(self.bias_regularizer),\n                  'kernel_constraint': constraints.serialize(self.kernel_constraint),\n                  'chain_constraint': constraints.serialize(self.chain_constraint),\n                  'boundary_constraint': constraints.serialize(self.boundary_constraint),\n                  'bias_constraint': constraints.serialize(self.bias_constraint),\n                  'input_dim': self.input_dim,\n                  'unroll': self.unroll}\n        base_config = super(CRF, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n\n    @property\n    def loss_function(self):\n        if self.learn_mode == 'join':\n            def loss(y_true, y_pred):\n                assert self._inbound_nodes, 'CRF has not connected to any layer.'\n                assert not self._outbound_nodes, 'When learn_model=\"join\", CRF must be the last layer.'\n                if self.sparse_target:\n                    y_true = K.one_hot(K.cast(y_true[:, :, 0], 'int32'), self.units)\n                X = self._inbound_nodes[0].input_tensors[0]\n                mask = self._inbound_nodes[0].input_masks[0]\n                nloglik = self.get_negative_log_likelihood(y_true, X, mask)\n                return nloglik\n            return loss\n        else:\n            if self.sparse_target:\n                return sparse_categorical_crossentropy\n            else:\n                return categorical_crossentropy\n\n    @property\n    def accuracy(self):\n        if self.test_mode == 'viterbi':\n            return self.viterbi_acc\n        else:\n            return self.marginal_acc\n\n    @staticmethod\n    def _get_accuracy(y_true, y_pred, mask, sparse_target=False):\n        y_pred = K.argmax(y_pred, -1)\n        if sparse_target:\n            y_true = K.cast(y_true[:, :, 0], K.dtype(y_pred))\n        else:\n            y_true = K.argmax(y_true, -1)\n        judge = K.cast(K.equal(y_pred, y_true), K.floatx())\n        if mask is None:\n            return K.mean(judge)\n        else:\n            mask = K.cast(mask, K.floatx())\n            return K.sum(judge * mask) / K.sum(mask)\n\n    @property\n    def viterbi_acc(self):\n        def acc(y_true, y_pred):\n            X = self._inbound_nodes[0].input_tensors[0]\n            mask = self._inbound_nodes[0].input_masks[0]\n            y_pred = self.viterbi_decoding(X, mask)\n            return self._get_accuracy(y_true, y_pred, mask, self.sparse_target)\n        acc.func_name = 'viterbi_acc'\n        return acc\n\n    @property\n    def marginal_acc(self):\n        def acc(y_true, y_pred):\n            X = self._inbound_nodes[0].input_tensors[0]\n            mask = self._inbound_nodes[0].input_masks[0]\n            y_pred = self.get_marginal_prob(X, mask)\n            return self._get_accuracy(y_true, y_pred, mask, self.sparse_target)\n        acc.func_name = 'marginal_acc'\n        return acc\n\n    @staticmethod\n    def softmaxNd(x, axis=-1):\n        m = K.max(x, axis=axis, keepdims=True)\n        exp_x = K.exp(x - m)\n        prob_x = exp_x / K.sum(exp_x, axis=axis, keepdims=True)\n        return prob_x\n\n    @staticmethod\n    def shift_left(x, offset=1):\n        assert offset > 0\n        return K.concatenate([x[:, offset:], K.zeros_like(x[:, :offset])], axis=1)\n\n    @staticmethod\n    def shift_right(x, offset=1):\n        assert offset > 0\n        return K.concatenate([K.zeros_like(x[:, :offset]), x[:, :-offset]], axis=1)\n\n    def add_boundary_energy(self, energy, mask, start, end):\n        start = K.expand_dims(K.expand_dims(start, 0), 0)\n        end = K.expand_dims(K.expand_dims(end, 0), 0)\n        if mask is None:\n            energy = K.concatenate([energy[:, :1, :] + start, energy[:, 1:, :]], axis=1)\n            energy = K.concatenate([energy[:, :-1, :], energy[:, -1:, :] + end], axis=1)\n        else:\n            mask = K.expand_dims(K.cast(mask, K.floatx()))\n            start_mask = K.cast(K.greater(mask, self.shift_right(mask)), K.floatx())\n            end_mask = K.cast(K.greater(self.shift_left(mask), mask), K.floatx())\n            energy = energy + start_mask * start\n            energy = energy + end_mask * end\n        return energy\n\n    def get_log_normalization_constant(self, input_energy, mask, **kwargs):\n        \"\"\"Compute logarithm of the normalization constant Z, where\n        Z = sum exp(-E) -> logZ = log sum exp(-E) =: -nlogZ\n        \"\"\"\n        # should have logZ[:, i] == logZ[:, j] for any i, j\n        logZ = self.recursion(input_energy, mask, return_sequences=False, **kwargs)\n        return logZ[:, 0]\n\n    def get_energy(self, y_true, input_energy, mask):\n        \"\"\"Energy = a1' y1 + u1' y1 + y1' U y2 + u2' y2 + y2' U y3 + u3' y3 + an' y3\n        \"\"\"\n        input_energy = K.sum(input_energy * y_true, 2)  # (B, T)\n        chain_energy = K.sum(K.dot(y_true[:, :-1, :], self.chain_kernel) * y_true[:, 1:, :], 2)  # (B, T-1)\n\n        if mask is not None:\n            mask = K.cast(mask, K.floatx())\n            chain_mask = mask[:, :-1] * mask[:, 1:]  # (B, T-1), mask[:,:-1]*mask[:,1:] makes it work with any padding\n            input_energy = input_energy * mask\n            chain_energy = chain_energy * chain_mask\n        total_energy = K.sum(input_energy, -1) + K.sum(chain_energy, -1)  # (B, )\n\n        return total_energy\n\n    def get_negative_log_likelihood(self, y_true, X, mask):\n        \"\"\"Compute the loss, i.e., negative log likelihood (normalize by number of time steps)\n           likelihood = 1/Z * exp(-E) ->  neg_log_like = - log(1/Z * exp(-E)) = logZ + E\n        \"\"\"\n        input_energy = self.activation(K.dot(X, self.kernel) + self.bias)\n        if self.use_boundary:\n            input_energy = self.add_boundary_energy(input_energy, mask, self.left_boundary, self.right_boundary)\n        energy = self.get_energy(y_true, input_energy, mask)\n        logZ = self.get_log_normalization_constant(input_energy, mask, input_length=K.int_shape(X)[1])\n        nloglik = logZ + energy\n        if mask is not None:\n            nloglik = nloglik / K.sum(K.cast(mask, K.floatx()), 1)\n        else:\n            nloglik = nloglik / K.cast(K.shape(X)[1], K.floatx())\n        return nloglik\n\n    def step(self, input_energy_t, states, return_logZ=True):\n        # not in the following  `prev_target_val` has shape = (B, F)\n        # where B = batch_size, F = output feature dim\n        # Note: `i` is of float32, due to the behavior of `K.rnn`\n        prev_target_val, i, chain_energy = states[:3]\n        t = K.cast(i[0, 0], dtype='int32')\n        if len(states) > 3:\n            if K.backend() == 'theano':\n                m = states[3][:, t:(t + 2)]\n            else:\n                m = K.tf.slice(states[3], [0, t], [-1, 2])\n            input_energy_t = input_energy_t * K.expand_dims(m[:, 0])\n            chain_energy = chain_energy * K.expand_dims(K.expand_dims(m[:, 0] * m[:, 1]))  # (1, F, F)*(B, 1, 1) -> (B, F, F)\n        if return_logZ:\n            energy = chain_energy + K.expand_dims(input_energy_t - prev_target_val, 2)  # shapes: (1, B, F) + (B, F, 1) -> (B, F, F)\n            new_target_val = K.logsumexp(-energy, 1)  # shapes: (B, F)\n            return new_target_val, [new_target_val, i + 1]\n        else:\n            energy = chain_energy + K.expand_dims(input_energy_t + prev_target_val, 2)\n            min_energy = K.min(energy, 1)\n            argmin_table = K.cast(K.argmin(energy, 1), K.floatx())  # cast for tf-version `K.rnn`\n            return argmin_table, [min_energy, i + 1]\n\n    def recursion(self, input_energy, mask=None, go_backwards=False, return_sequences=True, return_logZ=True, input_length=None):\n        \"\"\"Forward (alpha) or backward (beta) recursion\n        If `return_logZ = True`, compute the logZ, the normalization constant:\n        \\[ Z = \\sum_{y1, y2, y3} exp(-E) # energy\n          = \\sum_{y1, y2, y3} exp(-(u1' y1 + y1' W y2 + u2' y2 + y2' W y3 + u3' y3))\n          = sum_{y2, y3} (exp(-(u2' y2 + y2' W y3 + u3' y3)) sum_{y1} exp(-(u1' y1' + y1' W y2))) \\]\n        Denote:\n            \\[ S(y2) := sum_{y1} exp(-(u1' y1 + y1' W y2)), \\]\n            \\[ Z = sum_{y2, y3} exp(log S(y2) - (u2' y2 + y2' W y3 + u3' y3)) \\]\n            \\[ logS(y2) = log S(y2) = log_sum_exp(-(u1' y1' + y1' W y2)) \\]\n        Note that:\n              yi's are one-hot vectors\n              u1, u3: boundary energies have been merged\n        If `return_logZ = False`, compute the Viterbi's best path lookup table.\n        \"\"\"\n        chain_energy = self.chain_kernel\n        chain_energy = K.expand_dims(chain_energy, 0)  # shape=(1, F, F): F=num of output features. 1st F is for t-1, 2nd F for t\n        prev_target_val = K.zeros_like(input_energy[:, 0, :])  # shape=(B, F), dtype=float32\n\n        if go_backwards:\n            input_energy = K.reverse(input_energy, 1)\n            if mask is not None:\n                mask = K.reverse(mask, 1)\n\n        initial_states = [prev_target_val, K.zeros_like(prev_target_val[:, :1])]\n        constants = [chain_energy]\n\n        if mask is not None:\n            mask2 = K.cast(K.concatenate([mask, K.zeros_like(mask[:, :1])], axis=1), K.floatx())\n            constants.append(mask2)\n\n        def _step(input_energy_i, states):\n            return self.step(input_energy_i, states, return_logZ)\n\n        target_val_last, target_val_seq, _ = K.rnn(_step, input_energy, initial_states, constants=constants,\n                                                   input_length=input_length, unroll=self.unroll)\n\n        if return_sequences:\n            if go_backwards:\n                target_val_seq = K.reverse(target_val_seq, 1)\n            return target_val_seq\n        else:\n            return target_val_last\n\n    def forward_recursion(self, input_energy, **kwargs):\n        return self.recursion(input_energy, **kwargs)\n\n    def backward_recursion(self, input_energy, **kwargs):\n        return self.recursion(input_energy, go_backwards=True, **kwargs)\n\n    def get_marginal_prob(self, X, mask=None):\n        input_energy = self.activation(K.dot(X, self.kernel) + self.bias)\n        if self.use_boundary:\n            input_energy = self.add_boundary_energy(input_energy, mask, self.left_boundary, self.right_boundary)\n        input_length = K.int_shape(X)[1]\n        alpha = self.forward_recursion(input_energy, mask=mask, input_length=input_length)\n        beta = self.backward_recursion(input_energy, mask=mask, input_length=input_length)\n        if mask is not None:\n            input_energy = input_energy * K.expand_dims(K.cast(mask, K.floatx()))\n        margin = -(self.shift_right(alpha) + input_energy + self.shift_left(beta))\n        return self.softmaxNd(margin)\n\n    def viterbi_decoding(self, X, mask=None):\n        input_energy = self.activation(K.dot(X, self.kernel) + self.bias)\n        if self.use_boundary:\n            input_energy = self.add_boundary_energy(input_energy, mask, self.left_boundary, self.right_boundary)\n\n        argmin_tables = self.recursion(input_energy, mask, return_logZ=False)\n        argmin_tables = K.cast(argmin_tables, 'int32')\n\n        # backward to find best path, `initial_best_idx` can be any, as all elements in the last argmin_table are the same\n        argmin_tables = K.reverse(argmin_tables, 1)\n        initial_best_idx = [K.expand_dims(argmin_tables[:, 0, 0])]  # matrix instead of vector is required by tf `K.rnn`\n        if K.backend() == 'theano':\n            initial_best_idx = [K.T.unbroadcast(initial_best_idx[0], 1)]\n\n        def gather_each_row(params, indices):\n            n = K.shape(indices)[0]\n            if K.backend() == 'theano':\n                return params[K.T.arange(n), indices]\n            else:\n                indices = K.transpose(K.stack([K.tf.range(n), indices]))\n                return K.tf.gather_nd(params, indices)\n\n        def find_path(argmin_table, best_idx):\n            next_best_idx = gather_each_row(argmin_table, best_idx[0][:, 0])\n            next_best_idx = K.expand_dims(next_best_idx)\n            if K.backend() == 'theano':\n                next_best_idx = K.T.unbroadcast(next_best_idx, 1)\n            return next_best_idx, [next_best_idx]\n\n        _, best_paths, _ = K.rnn(find_path, argmin_tables, initial_best_idx, input_length=K.int_shape(X)[1], unroll=self.unroll)\n        best_paths = K.reverse(best_paths, 1)\n        best_paths = K.squeeze(best_paths, 2)\n\n        return K.one_hot(best_paths, self.units)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6d9ba67784a576692b6f38c931e3c012eff539d7"},"cell_type":"code","source":"def gru_crf(inp):\n    #K.clear_session()\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = SpatialDropout1D(rate=0.24)(x)\n    x = Bidirectional(CuDNNGRU(100, return_sequences=True, \n                                kernel_initializer=glorot_normal(seed=123000), recurrent_initializer=orthogonal(gain=1.0, seed=10000)))(x)\n\n    crf = CRF(10,learn_mode='marginal',unroll=True)\n    x = crf(x)\n\n    x = Flatten()(x)\n\n    x = Dense(100, activation='relu', kernel_initializer=glorot_normal(seed=123000))(x)\n    x = Dropout(0.2)(x)\n    x = BatchNormalization()(x)\n\n    x = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=x)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(),)\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b5a8d1d9ceff06d19f32ccefa62c601d08a4fd0c"},"cell_type":"code","source":"def f1_smart(y_true, y_pred):\n    args = np.argsort(y_pred)\n    tp = y_true.sum()\n    fs = (tp - np.cumsum(y_true[args[:-1]])) / np.arange(y_true.shape[0] + tp - 1, tp, -1)\n    res_idx = np.argmax(fs)\n    return 2 * fs[res_idx], (y_pred[args[res_idx]] + y_pred[args[res_idx + 1]]) / 2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0d97ed2a521ec1bd2fc10c3ec9fdb7c5fc6f571c"},"cell_type":"code","source":"kfold = StratifiedKFold(n_splits=5, random_state=10, shuffle=True)\nbestscore = []\ny_test = np.zeros((X_test.shape[0], ))\noof = np.zeros((X.shape[0], ))\nlogloss = []\nfor i, (train_index, valid_index) in enumerate(kfold.split(X, Y)):\n    K.clear_session()\n    X_train, X_val, Y_train, Y_val = X[train_index], X[valid_index], Y[train_index], Y[valid_index]\n    filepath=\"weights_best.h5\"\n    checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min')\n    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, min_lr=0.0001, verbose=2)\n    earlystopping = EarlyStopping(monitor='val_loss', min_delta=0.0001, patience=2, verbose=2, mode='auto')\n    callbacks = [checkpoint, reduce_lr]\n    model = gru_crf(Input(shape=(maxlen,)))\n    if i == 0:print(model.summary()) \n    model.fit(X_train, Y_train, batch_size=512, epochs=5, validation_data=(X_val, Y_val), verbose=2, callbacks=callbacks)\n    model.load_weights(filepath)\n    y_pred = model.predict([X_val], batch_size=1024, verbose=2)\n    logloss.append(np.min(model.history.history['val_loss']))\n    oof[valid_index] = np.squeeze(y_pred)\n    y_test += np.squeeze(model.predict([X_test], batch_size=1024, verbose=2))/5\n    f1, threshold = f1_smart(np.squeeze(Y_val), np.squeeze(y_pred))\n    print('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))\n    bestscore.append(threshold)\n    #if i == 1:\n    #break\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ee5440af2be57d6d16795bd2366576da0db7c653"},"cell_type":"code","source":"f1, threshold = f1_smart(np.squeeze(Y), oof)\nprint('Optimal F1: {:.4f} at threshold: {:.4f}'.format(f1, threshold))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f56a2cc5412e2f47efcdc7afef82b92f4477db0a"},"cell_type":"code","source":"np.mean(bestscore), np.mean(logloss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c75b99630279cdaf010878fb5b8d35be77765ad5"},"cell_type":"code","source":"y_test = y_test.reshape((-1, 1))\npred_test_y = (y_test>threshold).astype(int)\nsub['prediction'] = pred_test_y\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}