{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Meeting 3 - Finishing the Competition [Baseline]\n\nWelcome to meeting 3. This time we'll finish and wrap up the competition (Quora that is).\nIf any outsiders reach this, this is a baseline which we'll work upon in a Competence Group (NLP/ML) @ ÅF (malmö).\n"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \nimport regex as re\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport spacy\nimport unicodedata\nfrom sklearn.model_selection import StratifiedKFold, train_test_split\nfrom sklearn.utils.validation import check_X_y, check_is_fitted\nfrom sklearn import metrics\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import (Input, Embedding, SpatialDropout1D, Bidirectional,\n                          LSTM, GRU, GlobalMaxPool1D, Dense)\nfrom keras.models import Model\nimport tensorflow as tf\nfrom keras import backend as K\nfrom keras import initializers, regularizers, constraints, activations\nfrom keras.layers import Activation, Wrapper, InputSpec\nfrom keras.engine.topology import Layer\nfrom keras.callbacks import (EarlyStopping, ModelCheckpoint,\n                             ReduceLROnPlateau)\nfrom keras.utils.conv_utils import conv_output_length\nfrom tqdm import tqdm, tqdm_notebook\nimport operator\ntqdm(tqdm_notebook).pandas()\n#tqdm.pandas()\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\ndatapath='../input'\nRANDOM_STATE = 2\nSHUFFLE = True\nTEST_SIZE = 0.8\nTHRESHOLD = 0.35\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"import time\nfrom contextlib import contextmanager\n\n\n@contextmanager\ndef timer(name):\n    \"\"\"\n    Taken from Konstantin Lopuhin https://www.kaggle.com/lopuhin\n    in script named : Mercari Golf: 0.3875 CV in 75 LOC, 1900 s\n    https://www.kaggle.com/lopuhin/mercari-golf-0-3875-cv-in-75-loc-1900-s\n    \"\"\"\n    t0 = time.time()\n    yield\n    print(f'[{name}] done in {time.time() - t0:.0f} s')\n\n\ndef load_trained_model(model, weights_path):\n    model.load_weights(weights_path)\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data reader - An abstraction to data reading\nI've built an simple data reader class that'll help us to read data & we'll supply a transformer to transform & preprocess data within this\n\nUsage:\n```python\ndr = DataReader(train_file_path, module, test_file_path=None)\nsplit = dr.get_split(split=..)\nor\nsplit_generator = dr.get_kfold(k=..)\nsplit_K_0 = next(split_generator)\n```"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class DataReader(object):\n    def __init__(self,\n                 train_file,\n                 module,\n                 test_file=None):\n\n        if not train_file:\n            raise Exception(\"DataReader requires a train_file!\")\n        if not module:\n            raise Exception(\"DataReader requires a model that can transform data!\")\n        self.raw_test = None\n        if test_file:\n            print(\"Loading test_data (%s) into dataframe\" % test_file)\n            self.test_data = pd.read_csv(test_file)\n            self.raw_test = self.test_data[['question_text']]\n            print(\"Test data with shape: \", self.test_data.shape)\n\n        print(\"Loading train_data (%s) into dataframes\" % train_file)\n        self.train_data = pd.read_csv(train_file)\n        self.raw_train = self.train_data[['question_text']]\n        print(\"Train data with shape: \", self.train_data.shape)\n        train_test_cut = self.train_data.shape[0]\n        if isinstance(self.raw_test, pd.DataFrame):\n            df_all = pd.concat([self.raw_train, self.raw_test],\n                               axis=0).reset_index(drop=True)\n        else:\n            df_all = self.raw_train\n        self.df_all = df_all\n\n\n        print(\"Transforming the data\")\n        with timer('Transforming data'):\n            if module:\n                X_features = module.transform(df_all['question_text'])\n            else:\n                X_features = df_all['question_text']\n            # Multiple Inputs\n            if isinstance(X_features, list):\n                self.X_train = [X[:train_test_cut] for X in X_features]\n                self.X_test = [X[train_test_cut:] for X in X_features]\n            else:\n                self.X_train = X_features[:train_test_cut]\n                self.X_test = X_features[train_test_cut:]\n\n    def get_split(self, split=0.8, random_state=2, shuffle_data=True):\n        \"\"\"\n        :param split: float - % to be training data\n        :param random_state: int - init_state for random to keep random stale\n        :param shuffle_data: if to shuffle\n        :return: X_t, X_v, y_t, y_v where X = training and Y = validation.\n        t = training data & v = class\n        \"\"\"\n        print(\"Creating validation data by splitting (%s)\" % split)\n        train_data = self.train_data\n        X_train = self.X_train\n\n        X_t, X_v, y_t, y_v = train_test_split(\n            X_train, train_data.target,\n            test_size=(1 - split), random_state=random_state,\n            shuffle=shuffle_data, stratify=train_data.target)\n\n        return X_t, X_v, y_t, y_v\n\n    def get_kfold(self, k=5, shuffle_data=True, random_state=2):\n        \"\"\"\n        :param k: int - Number of folds.\n        :param shuffle_data: boolean - If we should shuffle\n        :param random_state: int - init_state for random to keep random stale\n        :return: a generator that yields the folds.\n        \"\"\"\n        print(\"Creating validation data by kfold (%s)\" % k)\n        kfold = StratifiedKFold(n_splits=k, shuffle=shuffle_data, random_state=random_state)\n        train_data = self.train_data\n        X_train = self.X_train\n        folded_data = kfold.split(X_train, train_data.target)\n\n        for i in range(k):\n            fold = next(folded_data)\n            X_t = X_train.iloc[fold[0]]\n            X_v = train_data.iloc[fold[0]]\n            y_t = X_train.iloc[fold[1]]\n            y_v = train_data.iloc[fold[1]]\n\n            yield X_t, X_v, y_t, y_v\n\n    def get_test(self):\n        if isinstance(self.test_data, pd.DataFrame):\n            return self.train_data, self.X_train, self.test_data, self.X_test\n        raise Exception(\"No test data provided!\")\n\n    def get_all_text(self):\n        return self.df_all['question_text']","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Preprocessor"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"class PreProcessor(object):\n    def __init__(self, text):\n        self.text = text\n        self.puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$',\n                       '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',\n                       '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`', '<',\n                       '→', '°', '€', '™', '›', '♥', '←', '×', '§', '″', '′', 'Â',\n                       '█', '½', 'à', '…', '“', '★', '”', '–', '●', 'â', '►', '−', '¢',\n                       '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥',\n                       '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’',\n                       '▀', '¨', '▄', '♫', '☆', 'é', '¯',\n                       '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞',\n                       '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³',\n                       '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']\n        # TODO this varies depending on what task!\n        self.mispell_dict = {'colour': 'color', 'centre': 'center', 'favourite': 'favorite', 'travelling': 'traveling',\n                             'counselling': 'counseling',\n                             'theatre': 'theater', 'cancelled': 'canceled', 'labour': 'labor',\n                             'organisation': 'organization',\n                             'wwii': 'world war 2',\n                             'citicise': 'criticize', 'youtu ': 'youtube ', 'Qoura': 'Quora', 'sallary': 'salary',\n                             'Whta': 'What',\n                             'narcisist': 'narcissist',\n                             'howdo': 'how do', 'whatare': 'what are', 'howcan': 'how can', 'howmuch': 'how much',\n                             'howmany': 'how many', 'whydo': 'why do',\n                             'doI': 'do I', 'theBest': 'the best', 'howdoes': 'how does',\n                             'mastrubation': 'masturbation',\n                             'mastrubate': 'masturbate',\n                             \"mastrubating\": 'masturbating', 'pennis': 'penis', 'Etherium': 'Ethereum',\n                             'narcissit': 'narcissist',\n                             'bigdata': 'big data',\n                             '2k17': '2017', '2k18': '2018', 'qouta': 'quota', 'exboyfriend': 'ex boyfriend',\n                             'airhostess': 'air hostess', \"whst\": 'what',\n                             'watsapp': 'whatsapp', 'demonitisation': 'demonetization',\n                             'demonitization': 'demonetization',\n                             'demonetisation': 'demonetization'}\n        self.mispellings_re = re.compile('(%s)' % '|'.join(self.mispell_dict.keys()))\n\n    def get_text(self):\n        return self.text\n\n    # TODO fix misspellings\n    def replace_typical_misspell(self):\n        def replace(match):\n            return self.mispell_dict[match.group(0)]\n\n        self.text = self.mispellings_re.sub(replace, self.text)\n\n        return self\n\n    def spacy_tokenize_words(self):\n        raise NotImplementedError\n\n    def normalize_unicode(self):\n        self.text = unicodedata.normalize('NFKD', self.text)\n        return self\n\n    def remove_newline(self):\n        \"\"\"\n        remove \\n and  \\t\n        \"\"\"\n        self.text = ' '.join(self.text.split())\n        return self\n\n    def decontracted(self):\n        # specific\n        text = re.sub(r\"(W|w)on(\\'|\\’)t\", \"will not\", self.text)\n        text = re.sub(r\"(C|c)an(\\'|\\’)t\", \"can not\", text)\n        text = re.sub(r\"(Y|y)(\\'|\\’)all\", \"you all\", text)\n        text = re.sub(r\"(Y|y)a(\\'|\\’)ll\", \"you all\", text)\n\n        # general\n        text = re.sub(r\"(I|i)(\\'|\\’)m\", \"i am\", text)\n        text = re.sub(r\"(A|a)in(\\'|\\’)t\", \"aint\", text)\n        text = re.sub(r\"n(\\'|\\’)t\", \" not\", text)\n        text = re.sub(r\"(\\'|\\’)re\", \" are\", text)\n        text = re.sub(r\"(\\'|\\’)s\", \" is\", text)\n        text = re.sub(r\"(\\'|\\’)d\", \" would\", text)\n        text = re.sub(r\"(\\'|\\’)ll\", \" will\", text)\n        text = re.sub(r\"(\\'|\\’)t\", \" not\", text)\n        self.text = re.sub(r\"(\\'|\\’)ve\", \" have\", text)\n\n        return self\n\n    def space_punctuation(self):\n        for punct in self.puncts:\n            if punct in self.text:\n                self.text = self.text.replace(punct, f' {punct} ')\n\n                # We could also remove all non p\\{L}...\n\n        return self\n\n    def remove_punctuation(self):\n        import string\n        re_tok = re.compile(f'([{string.punctuation}])')\n        self.text = re_tok.sub(' ', self.text)\n\n        return self\n\n    def clean_numbers(self):\n        text = self.text\n        if bool(re.search(r'\\d', text)):\n            text = re.sub('[0-9]{5,}', '#####', text)\n            text = re.sub('[0-9]{4}', '####', text)\n            text = re.sub('[0-9]{3}', '###', text)\n            text = re.sub('[0-9]{2}', '##', text)\n        self.text = text\n        return self\n\n    def clean_and_get_text(self):\n        self.clean_numbers() \\\n            .space_punctuation() \\\n            .decontracted() \\\n            .normalize_unicode() \\\n            .remove_newline() \\\n            .replace_typical_misspell()\n\n        return self.text","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Neural Network with Sklearn like interface\n\nIn the following file we've \n1. NeuralNetworkClassifier that is sklearn like interface (fit, train, predict & all you know! :))\n2. Scoring methods\n3. Attention\n4. Capsule\n5. DropConnect\n6. QRNN - A faster implementation (only Cuda!!) of RNN (Quasi Recurrent Neural Network)"},{"metadata":{"_kg_hide-input":false,"trusted":true},"cell_type":"code","source":"class NeuralNetworkClassifier:\n    \"\"\"\n    Neural Network classifier - sklearn like\n    \"\"\"\n\n    def __init__(self, model, batch_size=512, epochs=10, val_score='val_loss',\n                 reduce_lr=True, balancing_class_weight=False, filepath=None):\n        \"\"\"\n        Parameter\n        ---------\n        model: Keras model\n\n        batch_size: int or None, number of samples per gradient update\n\n        epochs: int, number of epochs to train the model\n\n        val_score: str, score to monitor. ['accuracy', 'precision_score',\n            'recall_score', 'f1_score', 'roc_auc_score']\n\n        reduce_lr: bool, if True, add a Keras callback function that\n            reduce learning rate when a metric has stopped improving\n\n        balancing_class_weight: bool, if True, uses the values of y to\n            automatically adjust weights inversely proportional to\n            class frequencies in the input data as\n            n_samples / (n_classes * np.bincount(y))\n\n        filepath: str, data directory that stores model pickle\n        \"\"\"\n        self.model = model\n        self.batch_size = batch_size\n        self.epochs = epochs\n        self.val_score = val_score\n        self.reduce_lr = reduce_lr\n        self.balancing_class_weight = balancing_class_weight\n        self.filepath = filepath\n        # compile model\n        self.model.compile(\n            loss='binary_crossentropy',\n            optimizer='adam',\n            metrics=['accuracy', precision_score, recall_score,\n                     f1_score, roc_auc_score])\n\n    def _get_class_weight(self, y):\n        # get class_weight\n        if self.balancing_class_weight:\n            from sklearn import utils\n            return utils.class_weight.compute_class_weight(\n                'balanced', np.unique(y), y)\n        else:\n            return None\n\n    def _get_callbacks(self):\n        callbacks = []\n        # get callbacks\n        callbacks.append(\n            EarlyStopping(\n                monitor=self.val_score,\n                patience=2,\n                verbose=1\n            )\n        )\n        if self.filepath:\n            callbacks.append(\n                ModelCheckpoint(\n                    filepath=self.filepath,\n                    monitor=self.val_score,\n                    save_best_only=True,\n                    save_weights_only=True\n                )\n            )\n        if self.reduce_lr:\n            callbacks.append(\n                ReduceLROnPlateau(\n                    monitor=self.val_score,\n                    factor=0.6,\n                    patience=1,\n                    min_lr=0.0001,\n                    verbose=2\n                )\n            )\n        return callbacks\n\n    def predict(self, X):\n        return (self.predict_proba(X) > THRESHOLD).astype(int)\n\n    def predict_proba(self, X):\n        return self.model.predict([X], batch_size=1024, verbose=1)\n\n    def train(self, X_train, y_train, X_val, y_val, verbose=1):\n        \"\"\"\n        train neural network and monitor the best iteration with validation\n\n        Parameters\n        ----------\n        X_train, y_train, X_val, y_val: features and targets\n\n        verbose: int, 0 = silent, 1 = progress bar, 2 = one line per epoch\n\n        Return\n        ------\n        self\n        \"\"\"\n        # get callbacks\n        callbacks = self._get_callbacks()\n        # get class_weight\n        class_weight = self._get_class_weight(y_train)\n        # train model\n        self.model.fit(\n            X_train, y_train,\n            batch_size=self.batch_size,\n            epochs=self.epochs,\n            verbose=verbose,\n            callbacks=callbacks,\n            validation_data=(X_val, y_val),\n            shuffle=True,\n            class_weight=class_weight)\n        return self\n\n    def fit(self, X, y, best_iteration=6, verbose=0):\n        \"\"\"\n        fit lightgbm with best iteration, which is the best model\n\n        Parameters\n        ----------\n        X, y: features and targets\n\n        best_iteration: int, optional (default=100),\n            number of boosting iterations\n\n        verbose: int, 0 = silent, 1 = progress bar, 2 = one line per epoch\n\n        Return\n        ------\n        self\n        \"\"\"\n        # get class_weight\n        class_weight = self._get_class_weight(y)\n        self.model.fit(\n            X, y,\n            batch_size=self.batch_size,\n            epochs=best_iteration,\n            verbose=verbose,\n            shuffle=True,\n            class_weight=class_weight)\n        # save model\n        if self.filepath:\n            self.model.save_weights(self.filepath)\n            print('saved fitted model to {}'.format(self.filepath))\n        return self\n\n    @property\n    def best_param(self):\n        scores = self.model.history.history[self.val_score]\n        if 'loss' in self.val_score:\n            func = min\n        else:\n            func = max\n        best_iteration, _ = func(enumerate(scores), key=operator.itemgetter(1))\n        return best_iteration + 1\n\n    @property\n    def best_score(self):\n        scores = self.model.history.history[self.val_score]\n        if 'loss' in self.val_score:\n            func = min\n        else:\n            func = max\n        _, best_val_f1 = func(enumerate(scores), key=operator.itemgetter(1))\n        return best_val_f1\n\n\n\"\"\"\nCustomized metrics during model training\n\"\"\"\n\n\ndef recall_score(y_true, y_proba, thres=THRESHOLD):\n    \"\"\"\n    Recall metric\n\n    Only computes a batch-wise average of recall\n\n    Computes the recall, a metric for multi-label classification of\n    how many relevant items are selected\n    \"\"\"\n    # get prediction\n    y_pred = K.cast(K.greater(y_proba, thres), dtype='float32')\n    # calc\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n    recall = true_positives / (possible_positives + K.epsilon())\n    return recall\n\n\ndef precision_score(y_true, y_proba, thres=THRESHOLD):\n    \"\"\"\n    Precision metric\n\n    Only computes a batch-wise average of precision\n\n    Computes the precision, a metric for multi-label classification of\n    how many selected items are relevant\n    \"\"\"\n    # get prediction\n    y_pred = K.cast(K.greater(y_proba, thres), dtype='float32')\n    # calc\n    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\n\ndef f1_score(y_true, y_proba, thres=THRESHOLD):\n    \"\"\"\n    F1 metric: geometric mean of precision and recall\n    \"\"\"\n    precision = precision_score(y_true, y_proba, thres)\n    recall = recall_score(y_true, y_proba, thres)\n    return 2 * ((precision * recall) / (precision + recall + K.epsilon()))\n\n\ndef roc_auc_score(y_true, y_proba):\n    \"\"\"\n    ROC AUC metric\n    \"\"\"\n    roc_auc = tf.metrics.auc(y_true, y_proba)[1]\n    K.get_session().run(tf.local_variables_initializer())\n    return roc_auc\n\n\n\"\"\"\nCustomized Keras layers for deep neural networks\n\"\"\"\n\n\nclass Attention(Layer):\n    \"\"\"\n    Keras Layer that implements an Attention mechanism for temporal data.\n    Supports Masking.\n    Follows the work of Raffel et al. [https://arxiv.org/abs/1512.08756]\n    # Input shape\n        3D tensor with shape: (samples, steps, features).\n    # Output shape\n        2D tensor with shape: (samples, features).\n    :param kwargs:\n    Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True. # noqa\n    The dimensions are inferred based on the output shape of the RNN.\n    Example:\n        model.add(LSTM(64, return_sequences=True))\n        model.add(Attention())\n    \"\"\"\n\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                              K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], self.features_dim\n\n\ndef squash(x, axis=-1):\n    s_squared_norm = K.sum(K.square(x), axis, keepdims=True)\n    scale = K.sqrt(s_squared_norm + K.epsilon())\n    return x / scale\n\n\nclass Capsule(Layer):\n    \"\"\"\n    Keras Layer that implements a Capsule for temporal data.\n    Literature publication: https://arxiv.org/abs/1710.09829v1\n    Youtube video introduction: https://www.youtube.com/watch?v=pPN8d0E3900\n    # Input shape\n        4D tensor with shape: (samples, steps, features).\n    # Output shape\n        3D tensor with shape: (samples, num_capsule, dim_capsule).\n    :param kwargs:\n    Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True. # noqa\n    The dimensions are inferred based on the output shape of the RNN.\n    Example:\n        model.add(\n            LSTM(\n                64,\n                return_sequences=True,\n                recurrent_initializer=orthogonal(gain=1.0, seed=10000)\n            )\n        )\n        model.add(\n            Capsule(\n                num_capsule=10,\n                dim_capsule=10,\n                routings=4,\n                share_weights=True\n            )\n        )\n    \"\"\"\n\n    def __init__(self, num_capsule, dim_capsule, routings=3,\n                 kernel_size=(9, 1), share_weights=True,\n                 activation='default', **kwargs):\n        super(Capsule, self).__init__(**kwargs)\n        self.num_capsule = num_capsule\n        self.dim_capsule = dim_capsule\n        self.routings = routings\n        self.kernel_size = kernel_size\n        self.share_weights = share_weights\n        if activation == 'default':\n            self.activation = squash\n        else:\n            self.activation = Activation(activation)\n\n    def build(self, input_shape):\n        super(Capsule, self).build(input_shape)\n        input_dim_capsule = input_shape[-1]\n        if self.share_weights:\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(1, input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),  # noqa\n                                     # shape=self.kernel_size,\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n        else:\n            input_num_capsule = input_shape[-2]\n            self.W = self.add_weight(name='capsule_kernel',\n                                     shape=(input_num_capsule,\n                                            input_dim_capsule,\n                                            self.num_capsule * self.dim_capsule),  # noqa\n                                     initializer='glorot_uniform',\n                                     trainable=True)\n\n    def call(self, u_vecs):\n        if self.share_weights:\n            u_hat_vecs = K.conv1d(u_vecs, self.W)\n        else:\n            u_hat_vecs = K.local_conv1d(u_vecs, self.W, [1], [1])\n\n        batch_size = K.shape(u_vecs)[0]\n        input_num_capsule = K.shape(u_vecs)[1]\n        u_hat_vecs = K.reshape(u_hat_vecs, (batch_size, input_num_capsule,\n                                            self.num_capsule, self.dim_capsule))  # noqa\n        u_hat_vecs = K.permute_dimensions(u_hat_vecs, (0, 2, 1, 3))\n        # final u_hat_vecs.shape = [None, num_capsule, input_num_capsule, dim_capsule]  # noqa\n\n        b = K.zeros_like(u_hat_vecs[:, :, :, 0])  # shape = [None, num_capsule, input_num_capsule]  # noqa\n        for i in range(self.routings):\n            b = K.permute_dimensions(b, (0, 2, 1))  # shape = [None, input_num_capsule, num_capsule]    # noqa\n            c = K.softmax(b)\n            c = K.permute_dimensions(c, (0, 2, 1))\n            b = K.permute_dimensions(b, (0, 2, 1))\n            outputs = self.activation(tf.keras.backend.batch_dot(c, u_hat_vecs, [2, 2]))  # noqa\n            if i < self.routings - 1:\n                b = tf.keras.backend.batch_dot(outputs, u_hat_vecs, [2, 3])\n        return outputs\n\n    def compute_output_shape(self, input_shape):\n        return (None, self.num_capsule, self.dim_capsule)\n\n\nclass DropConnect(Wrapper):\n    \"\"\"\n    Keras Wrapper that implements a DropConnect Layer.\n    When training with Dropout, a randomly selected subset of activations are\n    set to zero within each layer. DropConnect instead sets a randomly\n    selected subset of weights within the network to zero.\n    Each unit thus receives input from a random subset of units in the\n    previous layer.\n\n    Reference: https://cs.nyu.edu/~wanli/dropc/\n    Implementation: https://github.com/andry9454/KerasDropconnect\n    \"\"\"\n\n    def __init__(self, layer, prob, **kwargs):\n        self.prob = prob\n        self.layer = layer\n        super(DropConnect, self).__init__(layer, **kwargs)\n        if 0. < self.prob < 1.:\n            self.uses_learning_phase = True\n\n    def build(self, input_shape):\n        if not self.layer.built:\n            self.layer.build(input_shape)\n            self.layer.built = True\n        super(DropConnect, self).build()\n\n    def compute_output_shape(self, input_shape):\n        return self.layer.compute_output_shape(input_shape)\n\n    def call(self, x):\n        if 0. < self.prob < 1.:\n            self.layer.kernel = K.in_train_phase(\n                K.dropout(self.layer.kernel, self.prob),\n                self.layer.kernel)\n            self.layer.bias = K.in_train_phase(\n                K.dropout(self.layer.bias, self.prob),\n                self.layer.bias)\n        return self.layer.call(x, )\n\n\ndef _dropout(x, level, noise_shape=None, seed=None):\n    x = K.dropout(x, level, noise_shape, seed)\n    x *= (1. - level)  # compensate for the scaling by the dropout\n    return x\n\n\nclass QRNN(Layer):\n    '''Quasi RNN\n    # Arguments\n        units: dimension of the internal projections and the final output.\n    # References\n        - [Quasi-recurrent Neural Networks](http://arxiv.org/abs/1611.01576)\n    '''\n\n    def __init__(self, units, window_size=2, stride=1,\n                 return_sequences=False, go_backwards=False,\n                 stateful=False, unroll=False, activation='tanh',\n                 kernel_initializer='uniform', bias_initializer='zero',\n                 kernel_regularizer=None, bias_regularizer=None,\n                 activity_regularizer=None,\n                 kernel_constraint=None, bias_constraint=None,\n                 dropout=0, use_bias=True, input_dim=None, input_length=None,\n                 **kwargs):\n        self.return_sequences = return_sequences\n        self.go_backwards = go_backwards\n        self.stateful = stateful\n        self.unroll = unroll\n\n        self.units = units\n        self.window_size = window_size\n        self.strides = (stride, 1)\n\n        self.use_bias = use_bias\n        self.activation = activations.get(activation)\n        self.kernel_initializer = initializers.get(kernel_initializer)\n        self.bias_initializer = initializers.get(bias_initializer)\n        self.kernel_regularizer = regularizers.get(kernel_regularizer)\n        self.bias_regularizer = regularizers.get(bias_regularizer)\n        self.activity_regularizer = regularizers.get(activity_regularizer)\n        self.kernel_constraint = constraints.get(kernel_constraint)\n        self.bias_constraint = constraints.get(bias_constraint)\n\n        self.recurrent_dropout = 0  # not used, added to maintain compatibility with keras.Bidirectional\n        self.dropout = dropout\n        self.supports_masking = True\n        self.input_spec = [InputSpec(ndim=3)]\n        self.input_dim = input_dim\n        self.input_length = input_length\n        if self.input_dim:\n            kwargs['input_shape'] = (self.input_length, self.input_dim)\n        super(QRNN, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        if isinstance(input_shape, list):\n            input_shape = input_shape[0]\n\n        batch_size = input_shape[0] if self.stateful else None\n        self.input_dim = input_shape[2]\n        self.input_spec = InputSpec(shape=(batch_size, None, self.input_dim))\n        self.state_spec = InputSpec(shape=(batch_size, self.units))\n\n        self.states = [None]\n        if self.stateful:\n            self.reset_states()\n\n        kernel_shape = (self.window_size, 1, self.input_dim, self.units * 3)\n        self.kernel = self.add_weight(name='kernel',\n                                      shape=kernel_shape,\n                                      initializer=self.kernel_initializer,\n                                      regularizer=self.kernel_regularizer,\n                                      constraint=self.kernel_constraint)\n        if self.use_bias:\n            self.bias = self.add_weight(name='bias',\n                                        shape=(self.units * 3,),\n                                        initializer=self.bias_initializer,\n                                        regularizer=self.bias_regularizer,\n                                        constraint=self.bias_constraint)\n\n        self.built = True\n\n    def compute_output_shape(self, input_shape):\n        if isinstance(input_shape, list):\n            input_shape = input_shape[0]\n\n        length = input_shape[1]\n        if length:\n            length = conv_output_length(length + self.window_size - 1,\n                                        self.window_size, 'valid',\n                                        self.strides[0])\n        if self.return_sequences:\n            return (input_shape[0], length, self.units)\n        else:\n            return (input_shape[0], self.units)\n\n    def compute_mask(self, inputs, mask):\n        if self.return_sequences:\n            return mask\n        else:\n            return None\n\n    def get_initial_states(self, inputs):\n        # build an all-zero tensor of shape (samples, units)\n        initial_state = K.zeros_like(inputs)  # (samples, timesteps, input_dim)\n        initial_state = K.sum(initial_state, axis=(1, 2))  # (samples,)\n        initial_state = K.expand_dims(initial_state)  # (samples, 1)\n        initial_state = K.tile(initial_state, [1, self.units])  # (samples, units)\n        initial_states = [initial_state for _ in range(len(self.states))]\n        return initial_states\n\n    def reset_states(self, states=None):\n        if not self.stateful:\n            raise AttributeError('Layer must be stateful.')\n        if not self.input_spec:\n            raise RuntimeError('Layer has never been called '\n                               'and thus has no states.')\n\n        batch_size = self.input_spec.shape[0]\n        if not batch_size:\n            raise ValueError('If a QRNN is stateful, it needs to know '\n                             'its batch size. Specify the batch size '\n                             'of your input tensors: \\n'\n                             '- If using a Sequential model, '\n                             'specify the batch size by passing '\n                             'a `batch_input_shape` '\n                             'argument to your first layer.\\n'\n                             '- If using the functional API, specify '\n                             'the time dimension by passing a '\n                             '`batch_shape` argument to your Input layer.')\n\n        if self.states[0] is None:\n            self.states = [K.zeros((batch_size, self.units))\n                           for _ in self.states]\n        elif states is None:\n            for state in self.states:\n                K.set_value(state, np.zeros((batch_size, self.units)))\n        else:\n            if not isinstance(states, (list, tuple)):\n                states = [states]\n            if len(states) != len(self.states):\n                raise ValueError('Layer ' + self.name + ' expects ' +\n                                 str(len(self.states)) + ' states, '\n                                                         'but it received ' + str(len(states)) +\n                                 'state values. Input received: ' +\n                                 str(states))\n            for index, (value, state) in enumerate(zip(states, self.states)):\n                if value.shape != (batch_size, self.units):\n                    raise ValueError('State ' + str(index) +\n                                     ' is incompatible with layer ' +\n                                     self.name + ': expected shape=' +\n                                     str((batch_size, self.units)) +\n                                     ', found shape=' + str(value.shape))\n                K.set_value(state, value)\n\n    def __call__(self, inputs, initial_state=None, **kwargs):\n        # If `initial_state` is specified,\n        # and if it a Keras tensor,\n        # then add it to the inputs and temporarily\n        # modify the input spec to include the state.\n        if initial_state is not None:\n            if hasattr(initial_state, '_keras_history'):\n                # Compute the full input spec, including state\n                input_spec = self.input_spec\n                state_spec = self.state_spec\n                if not isinstance(state_spec, list):\n                    state_spec = [state_spec]\n                self.input_spec = [input_spec] + state_spec\n\n                # Compute the full inputs, including state\n                if not isinstance(initial_state, (list, tuple)):\n                    initial_state = [initial_state]\n                inputs = [inputs] + list(initial_state)\n\n                # Perform the call\n                output = super(QRNN, self).__call__(inputs, **kwargs)\n\n                # Restore original input spec\n                self.input_spec = input_spec\n                return output\n            else:\n                kwargs['initial_state'] = initial_state\n        return super(QRNN, self).__call__(inputs, **kwargs)\n\n    def call(self, inputs, mask=None, initial_state=None, training=None):\n        # input shape: `(samples, time (padded with zeros), input_dim)`\n        # note that the .build() method of subclasses MUST define\n        # self.input_spec and self.state_spec with complete input shapes.\n        if isinstance(inputs, list):\n            initial_states = inputs[1:]\n            inputs = inputs[0]\n        elif initial_state is not None:\n            pass\n        elif self.stateful:\n            initial_states = self.states\n        else:\n            initial_states = self.get_initial_states(inputs)\n\n        if len(initial_states) != len(self.states):\n            raise ValueError('Layer has ' + str(len(self.states)) +\n                             ' states but was passed ' +\n                             str(len(initial_states)) +\n                             ' initial states.')\n        input_shape = K.int_shape(inputs)\n        if self.unroll and input_shape[1] is None:\n            raise ValueError('Cannot unroll a RNN if the '\n                             'time dimension is undefined. \\n'\n                             '- If using a Sequential model, '\n                             'specify the time dimension by passing '\n                             'an `input_shape` or `batch_input_shape` '\n                             'argument to your first layer. If your '\n                             'first layer is an Embedding, you can '\n                             'also use the `input_length` argument.\\n'\n                             '- If using the functional API, specify '\n                             'the time dimension by passing a `shape` '\n                             'or `batch_shape` argument to your Input layer.')\n        constants = self.get_constants(inputs, training=None)\n        preprocessed_input = self.preprocess_input(inputs, training=None)\n\n        last_output, outputs, states = K.rnn(self.step, preprocessed_input,\n                                             initial_states,\n                                             go_backwards=self.go_backwards,\n                                             mask=mask,\n                                             constants=constants,\n                                             unroll=self.unroll,\n                                             input_length=input_shape[1])\n        if self.stateful:\n            updates = []\n            for i in range(len(states)):\n                updates.append((self.states[i], states[i]))\n            self.add_update(updates, inputs)\n\n        # Properly set learning phase\n        if 0 < self.dropout < 1:\n            last_output._uses_learning_phase = True\n            outputs._uses_learning_phase = True\n\n        if self.return_sequences:\n            return outputs\n        else:\n            return last_output\n\n    def preprocess_input(self, inputs, training=None):\n        if self.window_size > 1:\n            inputs = K.temporal_padding(inputs, (self.window_size - 1, 0))\n        inputs = K.expand_dims(inputs, 2)  # add a dummy dimension\n\n        output = K.conv2d(inputs, self.kernel, strides=self.strides,\n                          padding='valid',\n                          data_format='channels_last')\n        output = K.squeeze(output, 2)  # remove the dummy dimension\n        if self.use_bias:\n            output = K.bias_add(output, self.bias, data_format='channels_last')\n\n        if self.dropout is not None and 0. < self.dropout < 1.:\n            z = output[:, :, :self.units]\n            f = output[:, :, self.units:2 * self.units]\n            o = output[:, :, 2 * self.units:]\n            f = K.in_train_phase(1 - _dropout(1 - f, self.dropout), f, training=training)\n            return K.concatenate([z, f, o], -1)\n        else:\n            return output\n\n    def step(self, inputs, states):\n        prev_output = states[0]\n\n        z = inputs[:, :self.units]\n        f = inputs[:, self.units:2 * self.units]\n        o = inputs[:, 2 * self.units:]\n\n        z = self.activation(z)\n        f = f if self.dropout is not None and 0. < self.dropout < 1. else K.sigmoid(f)\n        o = K.sigmoid(o)\n\n        output = f * prev_output + (1 - f) * z\n        output = o * output\n\n        return output, [output]\n\n    def get_constants(self, inputs, training=None):\n        return []\n\n    def get_config(self):\n        config = {'units': self.units,\n                  'window_size': self.window_size,\n                  'stride': self.strides[0],\n                  'return_sequences': self.return_sequences,\n                  'go_backwards': self.go_backwards,\n                  'stateful': self.stateful,\n                  'unroll': self.unroll,\n                  'use_bias': self.use_bias,\n                  'dropout': self.dropout,\n                  'activation': activations.serialize(self.activation),\n                  'kernel_initializer': initializers.serialize(self.kernel_initializer),\n                  'bias_initializer': initializers.serialize(self.bias_initializer),\n                  'kernel_regularizer': regularizers.serialize(self.kernel_regularizer),\n                  'bias_regularizer': regularizers.serialize(self.bias_regularizer),\n                  'activity_regularizer': regularizers.serialize(self.activity_regularizer),\n                  'kernel_constraint': constraints.serialize(self.kernel_constraint),\n                  'bias_constraint': constraints.serialize(self.bias_constraint),\n                  'input_dim': self.input_dim,\n                  'input_length': self.input_length}\n        base_config = super(QRNN, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Embeddings & building of them"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def load_word_embedding(filepath):\n    \"\"\"\n    given a filepath to embeddings file, return a word to vec\n    dictionary, in other words, word_embedding\n\n    E.g. {'word': array([0.1, 0.2, ...])}\n    \"\"\"\n\n    def _get_vec(word, *arr):\n        return word, np.asarray(arr, dtype='float32')\n\n    print('Load word embeddings...')\n    try:\n        word_embedding = dict(_get_vec(*w.split(' ')) for w in open(filepath))\n    except UnicodeDecodeError:\n        word_embedding = dict(_get_vec(*w.split(' ')) for w in open(\n            filepath, encoding=\"utf8\", errors='ignore'))\n    # sanity check word vector length\n    words_to_del = []\n    for word, vec in word_embedding.items():\n        if len(vec) != 300:\n            words_to_del.append(word)\n    for word in words_to_del:\n        del word_embedding[word]\n    return word_embedding\n\n\ndef create_embedding_weights(word_index, word_embedding, max_features):\n    \"\"\"\n    create weights for embeddings layer where row is the word index\n    and collumns are the embeddings dense vector\n\n    Parameters\n    ----------\n    word_index: dict, mapping of word to word index. E.g. {'the': 2}\n        you can get word_index by keras.tokenizer.word_index\n\n    word_embedding: dict, mapping of word to word embeddings\n        E.g. {'the': array([0.1, 0.2, ...])}\n        you can get word_index by above function load_word_embedding and\n        embeddings filepath\n\n    max_features: int, number of words that we want to keep\n\n    Return\n    ------\n    embeddings weights: np.array, with shape (number of words, 300)\n    \"\"\"\n    print('Create word embeddings weights...')\n    # get entire embeddings matrix\n    mat_embedding = np.stack(word_embedding.values())\n    # get shape\n    a, b = min(max_features, len(word_index)), mat_embedding.shape[1]\n    print('Embeddings weights matrix with shape: ({}, {})'.format(a, b))\n    # init embeddings weight matrix\n    embedding_mean, embedding_std = mat_embedding.mean(), mat_embedding.std()\n    embedding_weights = np.random.normal(embedding_mean, embedding_std, (a, b))\n    # mapping\n    for word, idx in word_index.items():\n        if idx >= a:\n            continue\n        word_vec = word_embedding.get(word, None)\n        if word_vec is not None:\n            embedding_weights[idx] = word_vec\n    return embedding_weights","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Base Classifier - Neural Network"},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"import gc\nMAX_FEATURES = int(2.5e5)  # total word count = 227,538; clean word count = 186,551   # noqa\nMAX_LEN = 80    # mean_len = 12; Q99_len = 40; max_len = 189;\nRNN_UNITS = 40\nDENSE_UNITS_1 = 32\nDENSE_UNITS_2 = 16\nEMBED_FILEPATH = os.path.join(datapath, 'embeddings/glove.840B.300d/glove.840B.300d.txt')\nEMBED_PICKLEPATH = 'glove.pkl'\nMODEL_FILEPATH = 'submission.csv'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"code","source":"def get_network(embed_filepath):\n    input_layer = Input(shape=(MAX_LEN, ), name='input')\n    # 1. embeddings layer\n    # get embeddings weights\n    print('load pre-trained embeddings weights...')\n    embed_weights = pd.read_pickle(EMBED_PICKLEPATH)\n    input_dim = embed_weights.shape[0]\n    output_dim = embed_weights.shape[1]\n    x = Embedding(\n        input_dim=input_dim,\n        output_dim=output_dim,\n        weights=[embed_weights],\n        trainable=False,\n        name='embeddings'\n    )(input_layer)\n    # clean up\n    del embed_weights, input_dim, output_dim\n    gc.collect()\n    # 2. dropout\n    x = SpatialDropout1D(rate=0.15)(x)\n    # 3. bidirectional lstm & gru\n    x = Bidirectional(\n        layer=LSTM(RNN_UNITS, return_sequences=True),\n        name='bidirectional_lstm'\n    )(x)\n    #x = Bidirectional(\n    #    layer=GRU(RNN_UNITS, return_sequences=True),\n    #    name='bidirectional_gru'\n    #)(x)\n    # 4. global_max_pooling1d\n    x = GlobalMaxPool1D(name='global_max_pooling1d')(x)\n    # 5. dense\n    x = Dense(units=DENSE_UNITS_1, activation='relu', name='dense_1')(x)\n    x = Dense(units=DENSE_UNITS_2, activation='relu', name='dense_2')(x)\n    # 6. output (sigmoid)\n    output_layer = Dense(units=1, activation='sigmoid', name='output')(x)\n    return Model(inputs=input_layer, outputs=output_layer)\n\n\ndef get_model():\n    print('build network...')\n    model = get_network(embed_filepath=EMBED_FILEPATH)\n    print(model.summary())\n    return NeuralNetworkClassifier(\n        model,\n        balancing_class_weight=True,\n        filepath=MODEL_FILEPATH)\n\ndef tokenize(df_text):\n    # preprocess\n    def _clean(text):\n        return PreProcessor(text).clean_and_get_text()\n    df_text = df_text.progress_apply(_clean)\n    # tokenizer\n    tokenizer = Tokenizer(\n        num_words=MAX_FEATURES,\n        filters='',\n        lower=False,\n        split=' ')\n    # fit to data\n    tokenizer.fit_on_texts(list(df_text))\n    # tokenize the texts into sequences\n    sequences = tokenizer.texts_to_sequences(df_text)\n    return sequences, tokenizer\n\n\ndef transform(df_text):\n    seqs, _ = tokenize(df_text)\n    # pad the sentences\n    X = pad_sequences(seqs, maxlen=MAX_LEN, padding='pre', truncating='post')\n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class fakemodule(object):\n    @staticmethod\n    def transform(a):\n        return transform(a)\ndr = DataReader('%s/train.csv' % datapath, fakemodule, os.path.join(datapath, 'test.csv'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Create Embedding Matrix for Keras"},{"metadata":{"trusted":true},"cell_type":"code","source":"t0 = time.time()\nwith timer(\"Extract Word Index From Train and Test Data\"):\n    print(\"Loading data...\")\n    all_text = dr.get_all_text()\n    # get word index\n    print('Tokenizing text...')\n    _, tokenizer = tokenize(all_text)\n    word_index = tokenizer.word_index\n# 3. create embeddings weights matrix\nwith timer(\"Create Embedding Weights Matrix\"):\n    # load word embeddings\n    print('Loading embeddings file')\n    word_embed = load_word_embedding(EMBED_FILEPATH)\n    # create embeddings weights matrix\n    print('Create embeddings weights...')\n    embed_weights = create_embedding_weights(\n        word_index,\n        word_embed,\n        MAX_FEATURES)\n    # pickle numpy file\n    filepath_to_save = '{}.pkl'.format('glove')\n    pd.to_pickle(embed_weights, filepath_to_save)\n    print('Save embeddings weights to {}'.format(filepath_to_save))\n# record time spent\nprint('Entire program is done and it took {:.2f}s'.format(time.time() - t0))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Trainer"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_and_eval(X_train, y_train, X_val, y_val):\n    \"\"\"\n    Parameters\n    ----------\n    X_train, y_train, X_val, y_val: features and targets\n    \n    Return\n    ------\n    training logs\n    \"\"\"\n    model = get_model()\n    print('Training model...')\n    model = model.train(X_train, y_train, X_val, y_val)\n    best_param = model.best_param\n    best_score = model.best_score\n    print(\"Best param: {:.4f} with best score: {}\".format(best_param, best_score))\n    return pd.DataFrame({'best_param': [best_param], 'best_score': [best_score]})\n\nt0 = time.time()\n\nwith timer(\"Load and Preprocess\"):\n    X_t, X_v, y_t, y_v = dr.get_split(TEST_SIZE)\n\nwith timer('Training and Tuning'):\n    #df_score = train_and_eval(X_t, y_t, X_v, y_v)\n    filepath = os.path.join(datapath, 'trainer_baseline.csv')\n    # df_score.to_csv(filepath)\n    print('Save CV score file to {}'.format(filepath))\n\nprint('Entire program is done and it took {:.2f}s'.format(time.time() - t0))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_submission(X_train, y_train, X_test, df_test):\n    \"\"\"\n    train model with entire training data, predict test data,\n    and create submission file\n\n    Parameters\n    ----------\n    X_train, y_train, X_test: features and targets\n    df_test: dataframe, test data\n    module: a python module\n\n    Return\n    ------\n    df_summission\n    \"\"\"\n    model = get_model()\n    print('Training model...')\n    model = model.fit(X_train, y_train)\n    # predict\n    print('Predicting test...')\n    y_pred = np.squeeze(model.predict_proba(X_test) > 0.35).astype('int')\n    #y_pred = model.predict(X_test)\n    # create submission file\n    return pd.DataFrame({'qid': df_test.qid, 'prediction': y_pred})\n\nt0 = time.time()\n\nwith timer(\"Load and Preprocess\"):\n    # Only init if didn't run training.\n    # dr = DataReader(os.path.join(datapath, 'quora', 'train.csv'), fakemodule, os.path.join(datapath, 'quora', 'test.csv'))\n    df_train, X_train, df_test, X_test = dr.get_test()\n# 3. create submission file\nwith timer('Trainning and Creating Submission'):\n    filepath = 'submission.csv'\n    df_submission = create_submission(\n        X_train, df_train.target,\n        X_test, df_test)\n    df_submission.to_csv(filepath, index=False)\n    print('Save submission file to {}'.format(filepath))\n\nprint('Entire program is done and it took {:.2f}s'.format(time.time() - t0))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}