{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"from csv import DictReader\nfrom random import Random\nfrom collections import Counter\n\nimport numpy as np\n\nfrom matplotlib import pyplot as plt","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"f2cd49442708186b9a201f1df0de45bc461c0acc"},"cell_type":"code","source":"RANDOM = Random(42)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6375e90151f1fa6825c0fffba46fd52431b9196b"},"cell_type":"markdown","source":"## Load Data"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def load_samples(path):\n    with open(path) as f:\n        samples = list(DictReader(f))\n        for sample in samples:\n            sample['target'] = int(sample.get('target', -1))\n        return samples","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"c3c4db7c62ab4796c171f7174f2e6345b7d0ce3f"},"cell_type":"code","source":"def train_val_split(samples, split=0.2):\n    RANDOM.shuffle(samples)\n    n_val = int(len(samples) * split)\n    return samples[:-n_val], samples[-n_val:]","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"9d8941a4f539548786ac55a0405c98142d43bf92"},"cell_type":"code","source":"def undersample_majority(samples, factor=2.0):\n    classes, counts = np.unique([sample['target'] for sample in samples], return_counts=True)\n    minority_class_count = counts.min()\n    undersampled = []\n    for class_ in classes:\n        class_samples = [sample for sample in samples if sample['target'] == class_]\n        RANDOM.shuffle(class_samples)\n        undersampled.extend(class_samples[:int(minority_class_count * factor)])\n    return undersampled","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7aa1950951427c5084166755d6c7a6c3e374971f"},"cell_type":"code","source":"samples = load_samples('../input/train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6520f247d43cb0be94897b5ed13daa1ca81ea552"},"cell_type":"code","source":"train_samples, val_samples = train_val_split(samples)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b51eca3370a9f1e2e735f3892781d55923402212"},"cell_type":"markdown","source":"## Sampling"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"a81ab7b8d014d7d91af892c871309a6732fd1897"},"cell_type":"code","source":"plt.title('Target Distribution')\nplt.hist([sample['target'] for sample in train_samples])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0a91472b0fba9ee2cfa7082d9fbc54da16cee05f"},"cell_type":"markdown","source":"## Preprocessing"},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"759aba1d10c5942009ba1540b71604be7bdcbdcb"},"cell_type":"code","source":"def build_vocabulary(samples, vocab_min_freq=100):\n    counts = Counter(ch for sample in samples for ch in sample['question_text'])\n    chars = sorted(ch for ch, count in counts.items() if count >= vocab_min_freq)\n    return {char: i for i, char in enumerate(chars)}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cea3ed30248f4fa394441ca98296c1bb1de485ad"},"cell_type":"code","source":"vocabulary = build_vocabulary(train_samples)\nprint(len(vocabulary))\nvocabulary","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"trusted":true,"_uuid":"0399fafa54b3210aa5de3cc776943c2135776605"},"cell_type":"code","source":"def transform(sample, vocabulary):\n    sample['encoded_text'] = [vocabulary[ch] for ch in sample['question_text'] if ch in vocabulary]\n    return sample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3112df9441d7d951e0e759fb1fcca1d09d93048a"},"cell_type":"code","source":"train_samples = [transform(sample, vocabulary) for sample in train_samples]\nval_samples = [transform(sample, vocabulary) for sample in val_samples]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"_uuid":"37916b2339534638549dc343422244cf56bdc03e"},"cell_type":"code","source":"train_samples[0]","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0ccbddf1488f16c8842ff260969dd540ab7397a8"},"cell_type":"markdown","source":"## Modeling"},{"metadata":{"_kg_hide-input":false,"trusted":true,"_uuid":"77b407777c42a5df4ab0b175e25751f289f7f655"},"cell_type":"code","source":"from keras import backend as K\nfrom keras import initializers, regularizers, constraints\nfrom keras.engine import Layer\n\n\ndef dot_product(x, kernel):\n    \"\"\"\n    Wrapper for dot product operation, in order to be compatible with both\n    Theano and Tensorflow\n    Args:\n        x (): input\n        kernel (): weights\n    Returns:\n    \"\"\"\n    if K.backend() == 'tensorflow':\n        return K.squeeze(K.dot(x, K.expand_dims(kernel)), axis=-1)\n    else:\n        return K.dot(x, kernel)\n\n\nclass AttentionWithContext(Layer):\n    \"\"\"\n    Attention operation, with a context/query vector, for temporal data.\n    Supports Masking.\n    Follows the work of Yang et al. [https://www.cs.cmu.edu/~diyiy/docs/naacl16.pdf]\n    \"Hierarchical Attention Networks for Document Classification\"\n    by using a context vector to assist the attention\n    # Input shape\n        3D tensor with shape: `(samples, steps, features)`.\n    # Output shape\n        2D tensor with shape: `(samples, features)`.\n\n    How to use:\n    Just put it on top of an RNN Layer (GRU/LSTM/SimpleRNN) with return_sequences=True.\n    The dimensions are inferred based on the output shape of the RNN.\n\n    Note: The layer has been tested with Keras 2.0.6\n\n    Example:\n        model.add(LSTM(64, return_sequences=True))\n        model.add(AttentionWithContext())\n        # next add a Dense layer (for classification/regression) or whatever...\n    \"\"\"\n\n    def __init__(self,\n                 W_regularizer=None, u_regularizer=None, b_regularizer=None,\n                 W_constraint=None, u_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.u_regularizer = regularizers.get(u_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.u_constraint = constraints.get(u_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        super(AttentionWithContext, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1], input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        if self.bias:\n            self.b = self.add_weight((input_shape[-1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n\n        self.u = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_u'.format(self.name),\n                                 regularizer=self.u_regularizer,\n                                 constraint=self.u_constraint)\n\n        super(AttentionWithContext, self).build(input_shape)\n\n    def compute_mask(self, input, input_mask=None):\n        # do not pass the mask to the next layers\n        return None\n\n    def call(self, x, mask=None):\n        uit = dot_product(x, self.W)\n\n        if self.bias:\n            uit += self.b\n\n        uit = K.tanh(uit)\n        ait = dot_product(uit, self.u)\n\n        a = K.exp(ait)\n\n        # apply mask after the exp. will be re-normalized next\n        if mask is not None:\n            # Cast the mask to floatX to avoid float64 upcasting in theano\n            a *= K.cast(mask, K.floatx())\n\n        # in some cases especially in the early stages of training the sum may be almost zero\n        # and this results in NaN's. A workaround is to add a very small positive number ε to the sum.\n        # a /= K.cast(K.sum(a, axis=1, keepdims=True), K.floatx())\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], input_shape[-1]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b00234b1283b37886bb9e1f5307ed01a3a79ecb2"},"cell_type":"code","source":"from keras import models, Model\nfrom keras.layers import Input, Embedding, Conv1D, Add, Dense, SpatialDropout1D\n\ndef _conv_block(x, filters, kernel_size):\n    conv = Conv1D(filters, kernel_size, activation='relu', padding='same')(x)\n    conv = Conv1D(filters, kernel_size, activation='relu', padding='same')(conv)\n    return conv\n\n\ndef _resblock(x, filters, kernel_size):\n    conv = _conv_block(x, filters, kernel_size)\n    projection = Conv1D(filters, 1, padding='same')(x)\n    return Add()([conv, projection])\n\n\ndef troll_hunter(vocab_size,\n                 char_embedding_size,\n                 base_filters,\n                 doc_embedding_size,\n                 dropout):\n    text = Input(shape=(None,))\n    embedding = Embedding(vocab_size, char_embedding_size)(text)\n\n    conv_1 = _resblock(embedding, base_filters, 3)\n    conv_1 = SpatialDropout1D(dropout)(conv_1)\n    conv_2 = _resblock(conv_1, base_filters * 2, 3)\n    conv_2 = SpatialDropout1D(dropout)(conv_2)\n    conv_3 = _resblock(conv_2, base_filters * 4, 3)\n    conv_3 = SpatialDropout1D(dropout)(conv_3)\n    conv_4 = _resblock(conv_3, base_filters * 8, 3)\n    conv_4 = SpatialDropout1D(dropout)(conv_4)\n\n    attention = AttentionWithContext()(conv_4)\n\n    fc_1 = Dense(doc_embedding_size, activation='relu')(attention)\n    fc_2 = Dense(doc_embedding_size, activation='relu')(fc_1)\n    prediction = Dense(1, activation='sigmoid')(fc_2)\n\n    model = Model(text, prediction)\n    model.compile('adam', 'binary_crossentropy', metrics=['acc'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9c48deb6eb418c4e0fefb4321d8ec66d83633d72"},"cell_type":"markdown","source":"## Training Support"},{"metadata":{"trusted":true,"_uuid":"c80edd9ecc0469f7ba9b0b8e26f5d2c894f2e2ed"},"cell_type":"code","source":"from math import ceil\n\nclass BatchProvider:\n    def __init__(self, samples, batch_size, shuffle=False, run_forever=False):\n        self._samples = samples\n        self._batch_size = batch_size\n        self._shuffle = shuffle\n        self._run_forever = run_forever\n\n    def generate_batches(self):\n        batch = []\n        indices = list(range(len(self._samples)))\n        while True:\n            if self._shuffle:\n                RANDOM.shuffle(indices)\n            for i in indices:\n                batch.append(self.get_item(i))\n                if len(batch) == self._batch_size:\n                    yield self.transform_batch(batch)\n                    batch = []\n            if not self._run_forever:\n                break\n        if batch:\n            yield self.transform_batch(batch)\n\n    def __len__(self):\n        return int(ceil(len(self._samples) / self._batch_size))\n\n    def get_item(self, idx):\n        sample = self._samples[idx]\n        return sample['encoded_text'], sample['target']\n\n    def transform_batch(self, items):\n        texts, targets = zip(*items)\n        max_length = max(len(text) for text in texts)\n        text_batch = np.zeros((len(texts), max_length))\n        for i, text in enumerate(texts):\n            text_batch[i, :len(text)] = text\n        target_batch = np.array(targets)\n        return text_batch, target_batch","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"965761540885c8c9f94fbd18296ee2415edf8f3c"},"cell_type":"markdown","source":"## Training"},{"metadata":{"trusted":true,"_uuid":"e4dfbf9e3e87cad230b8f72719ce71e1e94303fb"},"cell_type":"code","source":"model = troll_hunter(\n    vocab_size=len(vocabulary),\n    char_embedding_size=16,\n    base_filters=32,\n    doc_embedding_size=300,\n    dropout=0.1\n)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1edff22ca083ce90322122f6766efccb3c51e397"},"cell_type":"code","source":"train_batch_provider = BatchProvider(train_samples, batch_size=64, shuffle=True, run_forever=True)\ntrain_batches = train_batch_provider.generate_batches()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"012e9b9f66fb77eedb99ecf64370658eba15d41e"},"cell_type":"code","source":"val_batch_provider = BatchProvider(val_samples, batch_size=64, shuffle=False, run_forever=True)\nval_batches = val_batch_provider.generate_batches()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4441866a3c94e2ca359fd9b87dfac1ea82e46e18"},"cell_type":"code","source":"from keras.callbacks import ModelCheckpoint, EarlyStopping\n\nmodel.fit_generator(\n        generator=train_batches,\n        steps_per_epoch=len(train_batch_provider),\n        epochs=5,\n        validation_data=val_batches,\n        validation_steps=len(val_batch_provider),\n        callbacks=[\n            ModelCheckpoint('weights.hdf5', monitor='val_acc', save_best_only=True),\n        ]\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7a900ae6d2f4acbb51cf6d575292a871b481b8eb"},"cell_type":"code","source":"model.load_weights('weights.hdf5')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad3b029bee337c9e6b24d333f231019d610888be"},"cell_type":"markdown","source":"## Calibration"},{"metadata":{"trusted":true,"_uuid":"f80b3326275604524a6279aec1b0e30e81812c1c"},"cell_type":"code","source":"from sklearn.metrics import precision_recall_curve\n\nval_batch_provider = BatchProvider(val_samples, batch_size=64, shuffle=False, run_forever=False)\nval_predictions = model.predict_generator(val_batch_provider.generate_batches(), steps=len(val_batch_provider))\nval_targets = np.array([sample['target'] for sample in val_samples])\n\nprecisions, recalls, thresholds = precision_recall_curve(val_targets, val_predictions[:, 0])\nprecisions = precisions[:-1]\nrecalls = recalls[:-1]\nf1_scores = 2 * ((precisions * recalls) / (precisions + recalls))\nthreshold = thresholds[f1_scores.argmax()]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d434acfed7a51f361983610f0fca33f6b0f2cb27"},"cell_type":"code","source":"print('f1: %f, threshold: %f' % (f1_scores.max(), threshold))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5bcb2fbc865437cb6cc563d21e67a6e95523ccda"},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true,"_uuid":"59d291977e9cfc894afd4daa7c499d2d36dba914"},"cell_type":"code","source":"from keras.utils import GeneratorEnqueuer\nimport csv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e5f67585f0930085a0a450aafe5046cc391bbec5"},"cell_type":"code","source":"test_samples = load_samples('../input/test.csv')\ntest_samples = [transform(sample, vocabulary) for sample in test_samples]\nsample_ids = (sample['qid'] for sample in test_samples)\n\ntest_batch_provider = BatchProvider(test_samples, batch_size=64, shuffle=False, run_forever=False)\nenqueuer = GeneratorEnqueuer(test_batch_provider.generate_batches())\nenqueuer.start()\ntest_batches = enqueuer.get()\n\nwith open('submission.csv', 'w') as submission_file:\n    submission_writer = csv.writer(submission_file)\n    submission_writer.writerow(['qid', 'prediction'])\n    for batch, _ in test_batches:\n        predictions = model.predict_on_batch(batch)\n        for prediction in predictions:\n            sample_id = next(sample_ids)\n            submission_writer.writerow([sample_id, 1 if prediction[0] >= threshold else 0])","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}