{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import keras\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn import metrics\n\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 60\nMAX_WORDS = 45000\nEMBEDDINGS_LOADED_DIMENSIONS = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80c188d28dac370261b2fba477f6f85f58756a94"},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1485d03a810b6f433541f7cf8ab373be895bd139"},"cell_type":"code","source":"BATCH_SIZE = 256\nQ_FRACTION = 1\nquestions = df_train.sample(frac=Q_FRACTION)\nquestion_texts = questions[\"question_text\"].values\nquestion_targets = questions[\"target\"].values\ntest_texts = df_test[\"question_text\"].fillna(\"_na_\").values\n\nprint(f\"Working on {len(questions)} questions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"adc1bbdf0874d110e69a484191ad5becc34e09bc"},"cell_type":"code","source":"def load_embeddings(file):\n    embeddings = {}\n    with open(file, encoding=\"utf8\", errors='ignore') as f:\n        def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n        embeddings = dict(get_coefs(*line.split(\" \")) for line in f)\n        \n    print('Found %s word vectors.' % len(embeddings))\n    return embeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dac5ea1dbf060a2618c9bab850e6daabd4473ba5"},"cell_type":"code","source":"from keras.preprocessing.text import Tokenizer\n\ntokenizer = Tokenizer(num_words=MAX_WORDS)\ntokenizer.fit_on_texts(list(df_train[\"question_text\"].values))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c559b21937fd5eeef51b4f367b781bb6638de40a"},"cell_type":"code","source":"from collections import defaultdict\n\ndef create_embedding_weights(tokenizer, embeddings, dimensions):\n    not_embedded = defaultdict(int)\n    \n    word_index = tokenizer.word_index\n    words_count = min(len(word_index), MAX_WORDS)\n    embeddings_matrix = np.zeros((words_count, dimensions))\n    for word, i in word_index.items():\n        if i >= MAX_WORDS:\n            continue\n        if word not in embeddings:\n            continue\n        embedding_vector = embeddings[word]\n        if embedding_vector is not None:\n            embeddings_matrix[i] = embedding_vector\n            \n    return embeddings_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef7fdf8d375e2fe22bf30aff536d8b753a073049"},"cell_type":"code","source":"THRESHOLD = 0.35\n\nclass EpochMetricsCallback(keras.callbacks.Callback):\n    def on_train_begin(self, logs={}):\n        self.f1s = []\n        self.precisions = []\n        self.recalls = []\n        \n    def on_epoch_end(self, epoch, logs={}):\n        predictions = self.model.predict(self.validation_data[0])\n        predictions = (predictions > THRESHOLD).astype(int)\n        predictions = np.asarray(predictions)\n        targets = self.validation_data[1]\n        f1 = metrics.f1_score(targets, predictions)\n        precision = metrics.precision_score(targets, predictions)\n        recall = metrics.recall_score(targets, predictions)\n\n        print(\" - F1 score: {0:.4f}, Precision: {1:.4f}, Recall: {2:.4f}\"\n              .format(f1, precision, recall))\n        self.f1s.append(f1)\n        self.precisions.append(precision)\n        self.recalls.append(recall)\n        return\n\nfrom matplotlib.ticker import MaxNLocator\n\ndef display_model_history(history):\n    data = pd.DataFrame(data={'Train': history.history['loss'], 'Test': history.history['val_loss']})\n    ax = sns.lineplot(data=data, palette=\"pastel\", linewidth=2.5, dashes=False)\n    ax.set(xlabel='Epoch', ylabel='Loss', title='Loss')\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n    sns.despine()\n    plt.show()\n\ndef display_model_epoch_metrics(epoch_callback):   \n    data = pd.DataFrame(data = {\n        'F1': epoch_callback.f1s,\n        'Precision': epoch_callback.precisions,\n        'Recall': epoch_callback.recalls})\n    ax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)\n    ax.set(xlabel='Epoch', title='Epoch metrics')\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n    sns.despine()\n    plt.show()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d8b71f90c1a7a0e05d028ad83a29c99d1ae3f9f9"},"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\n\nX = pad_sequences(tokenizer.texts_to_sequences(question_texts), maxlen=MAX_SEQUENCE_LENGTH)\nY = question_targets\n\ntest_word_tokens = pad_sequences(tokenizer.texts_to_sequences(test_texts), maxlen=MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e5e186594bc335e7cc2f77e1a9c1bd028a5ef27"},"cell_type":"code","source":"# Based on https://www.kaggle.com/hengzheng/attention-capsule-why-not-both-lb-0-694/notebook\nfrom keras.models import Sequential,Model\nfrom keras.layers import CuDNNLSTM, Dense, Bidirectional, Input,Dropout\n\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n        if self.bias:\n            eij += self.b\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b28f785b5a819fd7bec6a9a20f08946df068d470"},"cell_type":"code","source":"from keras.layers import Input, Embedding, Dense, Dropout, Flatten, BatchNormalization, SpatialDropout1D, SpatialDropout2D\nfrom keras.layers import LSTM, GRU, Bidirectional, CuDNNLSTM, CuDNNGRU\nfrom keras.models import Model\n\ndef make_model(emb_weights):\n    tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name=\"tokenized_input\")\n    embedding = Embedding(MAX_WORDS, EMBEDDINGS_LOADED_DIMENSIONS,\n                          weights=[emb_weights],\n                          trainable=False)(tokenized_input)\n    \n    embedding = SpatialDropout1D(0.15)(embedding)\n    lstm = Bidirectional(CuDNNGRU(64, return_sequences=True))(embedding)\n    lstm = SpatialDropout1D(0.15)(lstm)\n    lstm = Bidirectional(CuDNNGRU(32, return_sequences=True))(lstm)\n    a = Attention(MAX_SEQUENCE_LENGTH)(lstm)\n    d1 = Dense(32)(a)\n    d1 = Dropout(0.15)(d1)\n    b = BatchNormalization()(d1)\n    out = Dense(1, activation='sigmoid')(b)\n    \n    model = Model(inputs=[tokenized_input], outputs=out)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    model.summary()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8358a2ed2c7b15931d6d133734a6ea41c34d2a4b"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\ntrain_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.01)\n\n\ntest_predictions = []\nkaggle_predictions = []\n\nfrom gensim.models import KeyedVectors\n\nembedding_files = [\n    \"../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\",\n    \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\",\n    \"../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec\",\n    \"../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt\"]\n    \nload_embedding_functions = [ #\n#     lambda: KeyedVectors.load_word2vec_format(embedding_files[0], binary=True),\n    lambda: load_embeddings(embedding_files[1]),\n    lambda: load_embeddings(embedding_files[2]),\n    lambda: load_embeddings(embedding_files[3])]\n\nfor index, load_embeddings_fn in enumerate(load_embedding_functions):\n    print(f\"Training {embedding_files[index]}\")\n    pretrained_embeddings = load_embeddings_fn()\n    pretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS)\n    \n    epoch_callback = EpochMetricsCallback()\n    model = make_model(pretrained_emb_weights)\n    history = model.fit(x=train_X, y=train_Y, validation_split=0.015,\n                        batch_size=BATCH_SIZE, epochs=5, verbose=2,\n                        callbacks=[epoch_callback])\n\n    display_model_history(history)\n    display_model_epoch_metrics(epoch_callback)\n    \n    kaggle_predictions.append(model.predict([test_word_tokens], batch_size=1024, verbose=2))\n    test_predictions.append(model.predict([test_X]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a65ccd27d3b955cbde4e96c96c4329dff5bce350"},"cell_type":"code","source":"avg = np.average(kaggle_predictions, axis=0)\n\ndf_out = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\ndf_out['prediction'] = (avg > THRESHOLD).astype(int) \ndf_out.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65b47ad7f238e217b88e04216c4f11ce83e41dfe"},"cell_type":"code","source":"# Adjust the threshold\n\navg = np.average(test_predictions, axis=0)\nf1s = []\nprecisions = []\nrecalls = []\n\nTs = [x * 0.01 for x in range(20, 60)]\nfor t in Ts:\n    pred = (avg > t).astype(int)\n    f1s.append(metrics.f1_score(test_Y, pred))\n    precisions.append(metrics.precision_score(test_Y, pred))\n    recalls.append(metrics.recall_score(test_Y, pred))\n\n\ndata = pd.DataFrame(data = {'F1': f1s,\n                            'Precision': precisions,\n                            'Recall': recalls},\n                   index=Ts)\nax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)\nax.set(xlabel='Threshold', ylabel='Value', title='Threshold levels')\nsns.despine()\nplt.show()\n\nthresh = Ts[np.argmax(f1s)]\npred = (avg > thresh).astype(int)\nf1 = metrics.f1_score(test_Y, pred)\nprint(\"Test F1 {0:.4f} at threshold {1:.3f}\".format(f1, thresh))\n\nthresh = THRESHOLD\npred = (avg > thresh).astype(int)\nf1 = metrics.f1_score(test_Y, pred)\nprint(\"Test F1 {0:.4f} at threshold {1:.3f}\".format(f1, thresh))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"072cb6748adb5e30f2ae8e73585d99db88c25c80"},"cell_type":"code","source":"thresh = Ts[np.argmax(f1s)]\navg = np.average(kaggle_predictions, axis=0)\ndf_out = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\ndf_out['prediction'] = (avg > thresh).astype(int)\ndf_out.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"837fcb356e7915be1ff16d23ae00ff887447d6d2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}