{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import keras\nimport matplotlib.pyplot as plt\nimport numpy as np\nimport os\nimport pandas as pd\nimport seaborn as sns\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\n\nimport re\n\n\nprint(os.listdir(\"../input\"))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"MAX_SEQUENCE_LENGTH = 60\nMAX_WORDS = 45000\nEMBEDDINGS_LOADED_DIMENSIONS = 300","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"80c188d28dac370261b2fba477f6f85f58756a94"},"cell_type":"code","source":"df_train = pd.read_csv(\"../input/train.csv\")\ndf_test = pd.read_csv(\"../input/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1485d03a810b6f433541f7cf8ab373be895bd139"},"cell_type":"code","source":"BATCH_SIZE = 512\nQ_FRACTION = 1\nquestions = df_train.sample(frac=Q_FRACTION)\nquestion_texts = questions[\"question_text\"].values\nquestion_targets = questions[\"target\"].values\nkaggle_texts = df_test[\"question_text\"].fillna(\"_na_\").values\n\nprint(f\"Working on {len(questions)} questions\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"adc1bbdf0874d110e69a484191ad5becc34e09bc"},"cell_type":"code","source":"def load_embeddings(file):\n    embeddings = {}\n    with open(file, encoding=\"utf8\", errors='ignore') as f:\n        def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n        embeddings = dict(get_coefs(*line.split(\" \")) for (i, line) in enumerate(tqdm(f)))\n        \n    print('Found %s word vectors.' % len(embeddings))\n    return embeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c559b21937fd5eeef51b4f367b781bb6638de40a"},"cell_type":"code","source":"from collections import defaultdict\n\ndef create_embedding_weights(tokenizer, embeddings, dimensions):\n    not_embedded = defaultdict(int)\n    \n    word_index = tokenizer.word_index\n    words_count = min(len(word_index), MAX_WORDS)\n    embeddings_matrix = np.zeros((words_count, dimensions))\n    for word, i in word_index.items():\n        if i >= MAX_WORDS:\n            continue\n        if word not in embeddings:\n            continue\n        embedding_vector = embeddings[word]\n        if embedding_vector is not None:\n            embeddings_matrix[i] = embedding_vector\n            \n    return embeddings_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ef7fdf8d375e2fe22bf30aff536d8b753a073049"},"cell_type":"code","source":"THRESHOLD = 0.35\n\nclass EpochMetricsCallback(keras.callbacks.Callback):\n    def on_train_begin(self, logs={}):\n        self.f1s = []\n        self.precisions = []\n        self.recalls = []\n        \n    def on_epoch_end(self, epoch, logs={}):\n        predictions = self.model.predict(self.validation_data[0])\n        predictions = (predictions > THRESHOLD).astype(int)\n        predictions = np.asarray(predictions)\n        targets = self.validation_data[1]\n        f1 = metrics.f1_score(targets, predictions)\n        precision = metrics.precision_score(targets, predictions)\n        recall = metrics.recall_score(targets, predictions)\n\n        print(\" - F1 score: {0:.4f}, Precision: {1:.4f}, Recall: {2:.4f}\"\n              .format(f1, precision, recall))\n        self.f1s.append(f1)\n        self.precisions.append(precision)\n        self.recalls.append(recall)\n        return\n\nfrom matplotlib.ticker import MaxNLocator\n\ndef display_model_history(history):\n    data = pd.DataFrame(data={'Train': history.history['loss'], 'Test': history.history['val_loss']})\n    ax = sns.lineplot(data=data, palette=\"pastel\", linewidth=2.5, dashes=False)\n    ax.set(xlabel='Epoch', ylabel='Loss', title='Loss')\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n    sns.despine()\n    plt.show()\n\ndef display_model_epoch_metrics(epoch_callback):   \n    data = pd.DataFrame(data = {\n        'F1': epoch_callback.f1s,\n        'Precision': epoch_callback.precisions,\n        'Recall': epoch_callback.recalls})\n    ax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)\n    ax.set(xlabel='Epoch', title='Epoch metrics')\n    ax.xaxis.set_major_locator(MaxNLocator(integer=True))\n    sns.despine()\n    plt.show()\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"086daf46a8b1a225cfd927456b25eb580d12bf98"},"cell_type":"code","source":"from collections import defaultdict\nimport operator\nimport re\nfrom tqdm import tqdm\n\nclass Preprocessor:\n    def __init__(self, embeddings: dict):\n        self.embeddings = embeddings\n\n    def build_tf_dict(self, sentences: list):\n        \"\"\"\n        Build a simple TF (term frequency) dictionary for all words in the provided sentences.\n        \"\"\"\n        tf_dict = defaultdict(int)\n        for sentence in sentences:\n            for word in sentence:\n                tf_dict[word] += 1\n        return tf_dict\n\n    def check_coverage(self, tf_dictionary: dict):\n        \"\"\"\n        Build a simple list of words that are not embedded. Can be used down the stream to preprocess them to something\n        known.\n        \"\"\"\n        in_vocabulary = defaultdict(int)\n        out_of_vocabulary = defaultdict(int)\n        in_count = 0\n        out_count = 0\n\n        for word in tf_dictionary:\n            if word in self.embeddings:\n                in_vocabulary[word] = self.embeddings[word]\n                in_count += tf_dictionary[word]\n            else:\n                out_of_vocabulary[word] = tf_dictionary[word]\n                out_count += tf_dictionary[word]\n\n        percent_tf = len(in_vocabulary) / len(tf_dictionary)\n        percent_all = in_count / (in_count + out_count)\n        print('Found embeddings for {:.2%} of vocabulary and {:.2%} of all text'.format(percent_tf, percent_all))\n\n        return sorted(out_of_vocabulary.items(), key=operator.itemgetter(1))[::-1]\n\n    def clean_punctuation(self, text: list):\n        result = text\n        \n        for punct in \"/-'\":\n            result = result.replace(punct, ' ')\n        for punct in '&':\n            result = result.replace(punct, f' {punct} ')\n        for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n            result = result.replace(punct, '')\n\n        return result\n\n    def clean_digits(self, text: list):\n        result = text\n        result = re.sub('[0-9]{5,}', '#####', result)\n        result = re.sub('[0-9]{4}', '####', result)\n        result = re.sub('[0-9]{3}', '###', result)\n        result = re.sub('[0-9]{2}', '##', result)\n        return result\n\n    def clean_misspelling(self, text: list):\n        mispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium'}\n\n        def _get_mispell(mispell_dict):\n            mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n            return mispell_dict, mispell_re\n\n        mispellings, mispellings_re = _get_mispell(mispell_dict)\n\n        def replace(match):\n            return mispellings[match.group(0)]\n\n        return mispellings_re.sub(replace, text)\n    \n    def apply_cleaning_function(self, fn, texts: list, description = \"\"):\n        result = [fn(text) for text in texts]\n        sentences = [text.split() for text in result]\n        tf_dict = self.build_tf_dict(sentences)\n        oov = self.check_coverage(tf_dict)\n#         print(oov[:10])\n\n        return result\n\n    def preprocess_for_embeddings_coverage(self, texts: list):\n        result = texts\n\n        sentences = [text.split() for text in result]\n        tf_dict = self.build_tf_dict(sentences)\n        oov = self.check_coverage(tf_dict)\n\n        result = self.apply_cleaning_function(lambda x: self.clean_punctuation(x), result, \"Cleaning punctuation...\")\n#         result = self.apply_cleaning_function(lambda x: self.clean_digits(x), result, \"Cleaning numbers...\")\n#         result = self.apply_cleaning_function(lambda x: self.clean_misspelling(x), result, \"Cleaning misspelled words...\")\n\n        return result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e5e186594bc335e7cc2f77e1a9c1bd028a5ef27"},"cell_type":"code","source":"# Based on https://www.kaggle.com/hengzheng/attention-capsule-why-not-both-lb-0-694/notebook\nfrom keras.models import Sequential,Model\nfrom keras.layers import CuDNNLSTM, Dense, Bidirectional, Input,Dropout\n\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n        if self.bias:\n            eij += self.b\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b28f785b5a819fd7bec6a9a20f08946df068d470"},"cell_type":"code","source":"from keras.layers import Input, Embedding, Dense, Dropout, Flatten, BatchNormalization, SpatialDropout1D, SpatialDropout2D\nfrom keras.layers import LSTM, GRU, Bidirectional, CuDNNLSTM, CuDNNGRU\nfrom keras.models import Model\n\ndef make_model(emb_weights):\n    tokenized_input = Input(shape=(MAX_SEQUENCE_LENGTH,), name=\"tokenized_input\")\n    embedding = Embedding(MAX_WORDS, EMBEDDINGS_LOADED_DIMENSIONS,\n                          weights=[emb_weights],\n                          trainable=False)(tokenized_input)\n    \n    embedding = SpatialDropout1D(0.25)(embedding)\n    lstm = Bidirectional(CuDNNGRU(64, return_sequences=True))(embedding)\n    lstm = SpatialDropout1D(0.25)(lstm)\n    lstm = Bidirectional(CuDNNGRU(32, return_sequences=True))(lstm)\n    a = Attention(MAX_SEQUENCE_LENGTH)(lstm)\n    d1 = Dense(32)(a)\n    d1 = Dropout(0.25)(d1)\n    d1 = BatchNormalization()(d1)\n    out = Dense(1, activation='sigmoid')(d1)\n    \n    model = Model(inputs=[tokenized_input], outputs=out)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n#     model.summary()\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7e666f2b3372b5b215a22739fa19c3011d600c96"},"cell_type":"code","source":"from keras.preprocessing.sequence import pad_sequences\n\nX = question_texts\nY = question_targets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe94866e575b90ab7be4839a679c9d3bb741d502"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8358a2ed2c7b15931d6d133734a6ea41c34d2a4b"},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nfrom keras.preprocessing.text import Tokenizer\n\ntrain_X, test_X, train_Y, test_Y = train_test_split(X, Y, test_size=0.01)\n\ntest_predictions = []\nkaggle_predictions = []\n\nfrom gensim.models import KeyedVectors\n\nembedding_files = [\n    \"../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\",\n    \"../input/embeddings/glove.840B.300d/glove.840B.300d.txt\",\n    \"../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec\",\n    \"../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt\"]\n    \nload_embedding_functions = [\n#     lambda: KeyedVectors.load_word2vec_format(embedding_files[0], binary=True),\n    lambda: load_embeddings(embedding_files[1])]\n\nfor index, load_embeddings_fn in enumerate(load_embedding_functions):\n#     print(f\"Training with {embedding_files[index]}\")    \n    x_train = np.array(train_X)\n    y_train = np.array(train_Y)\n    \n    pretrained_embeddings = load_embeddings_fn()\n    preprocessor = Preprocessor(pretrained_embeddings)\n    \n    x_train = preprocessor.preprocess_for_embeddings_coverage(x_train)\n    x_kaggle = preprocessor.preprocess_for_embeddings_coverage(kaggle_texts)\n    x_test = preprocessor.preprocess_for_embeddings_coverage(test_X)\n    \n    tokenizer = Tokenizer(num_words=MAX_WORDS)\n    tokenizer.fit_on_texts(x_train)\n    \n    x_train = pad_sequences(tokenizer.texts_to_sequences(x_train), maxlen=MAX_SEQUENCE_LENGTH)\n    x_kaggle = pad_sequences(tokenizer.texts_to_sequences(x_kaggle), maxlen=MAX_SEQUENCE_LENGTH)\n    x_test = pad_sequences(tokenizer.texts_to_sequences(x_test), maxlen=MAX_SEQUENCE_LENGTH)\n    \n    pretrained_emb_weights = create_embedding_weights(tokenizer, pretrained_embeddings, EMBEDDINGS_LOADED_DIMENSIONS)\n    model = make_model(pretrained_emb_weights)\n        \n    splits = list(StratifiedKFold(n_splits=10, shuffle=True, random_state=317).split(x_train, y_train))\n    for idx, (train_idx, valid_idx) in enumerate(splits):\n        x = np.array(x_train[train_idx])\n        y = np.array(y_train[train_idx])\n        X_val = np.array(x_train[valid_idx])\n        y_val = np.array(y_train[valid_idx])\n    \n        epoch_callback = EpochMetricsCallback()\n        history = model.fit(x=x, y=y, validation_data=(X_val, y_val),\n                            batch_size=BATCH_SIZE, epochs=2, verbose=2,\n                            callbacks=[epoch_callback])\n\n        display_model_history(history)\n        display_model_epoch_metrics(epoch_callback)\n        \n    kaggle_predictions.append(model.predict([x_kaggle], batch_size=1024, verbose=2))\n    test_predictions.append(model.predict([x_test], batch_size=1024, verbose=2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a65ccd27d3b955cbde4e96c96c4329dff5bce350"},"cell_type":"code","source":"avg = np.average(kaggle_predictions, axis=0)\n\ndf_out = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\ndf_out['prediction'] = (avg > THRESHOLD).astype(int) \ndf_out.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65b47ad7f238e217b88e04216c4f11ce83e41dfe"},"cell_type":"code","source":"# Adjust the threshold\n\navg = np.average(test_predictions, axis=0)\nf1s = []\nprecisions = []\nrecalls = []\n\nTs = [x * 0.01 for x in range(25, 55)]\nfor t in Ts:\n    pred = (avg > t).astype(int)\n    f1s.append(metrics.f1_score(test_Y, pred))\n    precisions.append(metrics.precision_score(test_Y, pred))\n    recalls.append(metrics.recall_score(test_Y, pred))\n\n\ndata = pd.DataFrame(data = {'F1': f1s,\n                            'Precision': precisions,\n                            'Recall': recalls},\n                   index=Ts)\nax = sns.lineplot(data=data, palette='muted', linewidth=2.5, dashes=False)\nax.set(xlabel='Threshold', ylabel='Value', title='Threshold levels')\nsns.despine()\nplt.show()\n\nthresh = Ts[np.argmax(f1s)]\npred = (avg > thresh).astype(int)\nf1 = metrics.f1_score(test_Y, pred)\nprint(\"Test F1 {0:.4f} at threshold {1:.3f}\".format(f1, thresh))\n\nthresh = THRESHOLD\npred = (avg > thresh).astype(int)\nf1 = metrics.f1_score(test_Y, pred)\nprint(\"Test F1 {0:.4f} at threshold {1:.3f}\".format(f1, thresh))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"072cb6748adb5e30f2ae8e73585d99db88c25c80"},"cell_type":"code","source":"thresh = Ts[np.argmax(f1s)]\navg = np.average(kaggle_predictions, axis=0)\ndf_out = pd.DataFrame({\"qid\":df_test[\"qid\"].values})\ndf_out['prediction'] = (avg > thresh).astype(int)\ndf_out.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"837fcb356e7915be1ff16d23ae00ff887447d6d2"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}