{"cells":[{"metadata":{},"cell_type":"markdown","source":"This project comes from [this Kaggle competition](https://www.kaggle.com/c/quora-insincere-questions-classification). The aim of the project is to build a prediction model to identify whether a Quora question is insincere. The definition of an insincere question can be found [here](https://www.kaggle.com/c/quora-insincere-questions-classification/data)."},{"metadata":{},"cell_type":"markdown","source":"**Import Packages**"},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport tensorflow as tf\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import EarlyStopping\nfrom tensorflow.keras import backend as K\nfrom tensorflow.python.client.device_lib import list_local_devices\n\nfrom sklearn.model_selection import train_test_split, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score, precision_score, recall_score\n\nfrom nltk.stem import PorterStemmer, SnowballStemmer\nfrom nltk.stem.lancaster import LancasterStemmer\n\nimport os, gc\nimport io\nimport csv\nimport unicodedata\nfrom functools import partial\nfrom itertools import islice\nfrom hyperopt import hp, fmin, tpe, space_eval, Trials\nfrom tqdm import tqdm\nimport spacy\nfrom zipfile import ZipFile","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Global Variable Definitions**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Paths of input files\nINPUT_DIR = \"../input/\"\n\nPATH_TRAIN = os.path.join(INPUT_DIR, \"train.csv\")\nPATH_TEST = os.path.join(INPUT_DIR, \"test.csv\")\nPATH_RESULT = \"submission.csv\"\n\n# Model Constants\nMAX_SEQ_LEN = 50\nMAX_NUM_WORDS = None # 7*10**4\nLSTMLayer = CuDNNLSTM if any(d.device_type == \"GPU\" for d in list_local_devices()) else LSTM\nNUM_EPOCH = 20\n\nRANDOM_SEED = 0","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Functions for Performing Text Preprocessing and Feature Extraction**"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"def preprocess(df):\n    \"\"\"\n    Prepreprocess question texts \n    \"\"\"\n    texts = df[\"question_text\"].str.replace(r\"\\[math\\].+\\[/math\\]\", \"math formula\")\n    texts = texts.apply(partial(unicodedata.normalize, \"NFKD\"))\n    texts = texts.str.replace(\"[’`]\", '\\'')\n    texts = texts.str.replace(\"[“”„]\", '\"')\n    df[\"question_text\"] = texts\n    del texts\n    gc.collect()\n\ndef fit_tokenizer(*dfs, max_num_words=None):\n    \"\"\"\n    Fitting tokenizer from question texts\n    \"\"\"\n    if max_num_words:\n        tokenizer = Tokenizer(num_words=max_num_words + 2, oov_token=\"<UNK>\")\n    else:\n        tokenizer = Tokenizer(oov_token=\"<UNK>\")\n    \n    for df in dfs:\n        tokenizer.fit_on_texts(df[\"question_text\"])\n    print(f\"Number of tokenizer words: {len(tokenizer.word_index)}\")\n    return tokenizer\n\ndef compute_model_input(df, tokenizer):\n    \"\"\"\n    Compute model input from df using provided tokenizer\n    \"\"\"\n    texts = df[\"question_text\"]\n    sequences = tokenizer.texts_to_sequences(texts)\n    encoded_text = pad_sequences(sequences, padding=\"post\", maxlen=MAX_SEQ_LEN)\n    \n    df_features = pd.DataFrame(index=df.index)\n    df_features[\"seq_len\"] = [*map(len, sequences)]\n    df_features[\"num_qmarks\"] = [s.count('?') for s in texts]\n    df_features[\"num_commas\"] = [s.count(',') for s in texts]\n\n    word_index = tokenizer.word_index\n    qwords = [\"how\", \"what\", \"when\", \"where\", \"which\", \"who\", \"whom\", \"whose\", \"why\"]\n    qwords.extend([w.capitalize() for w in qwords])\n    qword_index = {filter(bool, map(word_index.get, qwords))}\n#     print(tokenizer.word_docs)\n    df_features[\"cnt_qwords\"] = [any(w in qword_index for w in seq) for seq in sequences]\n\n    return encoded_text, df_features","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Function for Loading Embedding Matrix**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class EmbeddingLoader:\n    \"\"\"\n    Utility class for loading embeddings\n    \"\"\"\n    STEM_FUNCS = [SnowballStemmer(\"english\").stem, PorterStemmer().stem, LancasterStemmer().stem]    \n    EMBEDDING_FILE = \"../input/embeddings.zip\"\n    PATH_DICT = {\"glove\": \"glove.840B.300d/glove.840B.300d.txt\",\n                 \"fasttext\": \"wiki-news-300d-1M/wiki-news-300d-1M.vec\",\n                 \"google-news\": \"GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin\"}\n    DIM_EMBEDDING = 300\n\n    @staticmethod\n    def load(name, tokenizer):\n        \"\"\"\n        Load word embeddings\n        \"\"\"\n        word_count = tokenizer.num_words\n        if not word_count:\n            word_count = len(tokenizer.word_index)\n        \n        with ZipFile(EmbeddingLoader.EMBEDDING_FILE) as embeddings_file:\n            with embeddings_file.open(EmbeddingLoader.PATH_DICT[name]) as embedding_file:\n                embedding = pd.read_csv(embedding_file, sep=' ', index_col=0, header=None,\n                                        dtype={i: np.float32 for i in range(1, EmbeddingLoader.DIM_EMBEDDING + 1)},\n                                        quoting=csv.QUOTE_NONE)\n        embedding_words = embedding.index\n        embedding_accessor = embedding.loc\n        print(f\"Loaded embedding file for {name}. Computing embedding matrix.\")\n        \n        word_count = tokenizer.num_words\n        if word_count:\n            df_word_idx = pd.DataFrame(index=[\"\", *islice(tokenizer.word_index, word_count)])\n        else:\n            df_word_idx = pd.DataFrame(index=[\"\", *tokenizer.word_index])\n        df_embedding_matrix = df_word_idx.join(embedding, how=\"left\").reindex(df_word_idx.index)\n        \n        df_uncovered = df_embedding_matrix.iloc[2:, 1].isna()\n        df_uncovered_idx = df_uncovered[df_uncovered].index\n        print(f\"{len(df_uncovered_idx)} out-of-vocabulary (OOV) words\")\n        df_stemmed = pd.DataFrame(None, index=df_uncovered_idx)\n        df_stemmed[\"stemmed\"] = [s.strip(\"'“”„ \") for s in df_uncovered_idx]\n        df_stemmed[\"covered\"] = df_stemmed[\"stemmed\"].isin(embedding_words)\n        \n        for stem_func in EmbeddingLoader.STEM_FUNCS:\n            df_uncovered_idx = df_stemmed[~df_stemmed[\"covered\"]].index\n            stemmed = df_uncovered_idx.map(stem_func)\n            df_stemmed.loc[df_uncovered_idx, \"stemmed\"] = stemmed\n            df_stemmed.loc[df_uncovered_idx, \"covered\"] = stemmed.isin(embedding_words)\n        \n        df_uncovered_idx = df_stemmed[~df_stemmed[\"covered\"]].index\n        df_stemmed.loc[df_uncovered_idx, \"stemmed\"] = df_uncovered_idx.str.capitalize()\n        df_embedding_matrix.fillna(df_stemmed.join(embedding, on=\"stemmed\", how=\"inner\"), inplace=True)\n        print(f\"After processing: {df_embedding_matrix[1].isna().sum()} OOV words\")\n        embedding_matrix = df_embedding_matrix.fillna(0).to_numpy()\n        \n        print(f\"Embedding matrix computed (shape: {embedding_matrix.shape}).\")\n        del embedding, df_word_idx, df_embedding_matrix, df_stemmed, df_uncovered, df_uncovered_idx\n        gc.collect()\n        \n        return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Attention Layer**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class Attention(Layer):\n    \"\"\"\n    Attention layer\n    \"\"\"\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 has_bias=True, **kwargs):\n        from tensorflow.keras import initializers, regularizers, constraints\n        \n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.has_bias = has_bias\n        self.step_dim = step_dim\n        self.feature_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.feature_dim = input_shape[-1]\n\n        if self.has_bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        feature_dim = self.feature_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, feature_dim)),\n                        K.reshape(self.W, (feature_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0], self.feature_dim","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model Building Function**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(input_data, embedding_weights, hparams):\n    \"\"\"\n    Build model using Keras API\n    \"\"\"\n    dim_feature = input_data[1].shape[1]\n    \n    input_text = Input((MAX_SEQ_LEN, ), name=\"TextInput\")\n    input_features = Input((dim_feature, ), name=\"FeatureInput\")\n    \n#     for embedding_weights in embeddings_weights:\n    X1 = Embedding(*embedding_weights.shape, input_length=MAX_SEQ_LEN,\n                   weights=[embedding_weights], trainable=False)(input_text)  # mask_zero=True\n    X1 = Bidirectional(LSTMLayer(128, return_sequences=True))(X1)\n#     X1 = Attention(3)(X1)\n#     X1 = Bidirectional(LSTMLayer(128, return_sequences=True))(X1)\n    X1 = Conv1D(64, 3, activation='relu')(X1)\n    X1 = GlobalMaxPool1D()(X1)\n    X1 = BatchNormalization()(X1)\n    X1 = Dense(32, activation='relu', name=\"dense1\")(X1)\n    X2 = Dense(4, activation='relu', name=\"dense_features\")(input_features)\n    X = concatenate([X1, X2])\n    X = Dense(8, activation='relu', name=\"dense2\")(X)\n    X = BatchNormalization()(X)\n    output = Dense(1, activation=\"sigmoid\", name=\"output\")(X)\n\n    model = Model(inputs=[input_text, input_features], outputs=output, name=\"QuestionClassificationModel\")\n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Model training and prediction functions**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_model(input_data, target, embedding_weights, hparams):\n    \"\"\"\n    Model building and training\n    \"\"\"\n    model = build_model(input_data, embedding_weights, hparams)\n    \n    if hparams[\"early_stopping\"]:\n        callbacks = [EarlyStopping(monitor='val_loss', min_delta=0.001, patience=10)]\n        val_split = 0.1\n    else:\n        callbacks = None\n        val_split = 0\n    \n    optimizer = Adam(learning_rate=hparams[\"learning_rate\"])\n    model.compile(loss=\"binary_crossentropy\", optimizer=optimizer, metrics=[\"accuracy\"])\n\n    model.fit(input_data, target, validation_split=val_split, callbacks=callbacks, \n              epochs=NUM_EPOCH, batch_size=hparams[\"batch_size\"],\n              class_weight={0: hparams[\"neg_class_weight\"], 1: 1}, shuffle=False, verbose=2)\n    return model\n\ndef compute_predictions(model, input_data, hparams):\n    \"\"\"\n    Compute predictions\n    \"\"\"\n    y_prob = model.predict(input_data)\n    return y_prob > hparams[\"pos_threshold\"]\n\ndef run_cross_validation(input_data, target, embedding_weights, hparams, kfold=5):\n    \"\"\"\n    Perform cross validation using provided input data, labels, as well as hyperparameters\n    \"\"\"\n\n    splitter = StratifiedKFold(kfold)\n    texts, features = input_data\n    predictions = np.zeros(len(target), dtype=np.bool)\n    \n    for fold_idx, (train_index, test_index) in enumerate(splitter.split(features, target)):\n        print(f\"Running on Fold {fold_idx}\")\n        input_train = texts[train_index], features.iloc[train_index]\n        target_train = target[train_index]\n        model = train_model(input_train, target_train, embedding_weights, hparams)\n        input_test = texts[test_index], features.iloc[test_index]\n        predictions[test_index] = compute_predictions(model, input_test, hparams)[0]\n    \n    score = f1_score(target, predictions)\n    print(f\"F1 Score: {score}\")\n    return score","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Pipeline**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def run(test_output_path, tune_params=True):\n    \"\"\"\n    Model training, \n    \"\"\"\n    HP_TUNING_EVALS = 100  # Number of evaluations in hyperparameter tuning\n    \n    df_train = pd.read_csv(PATH_TRAIN, usecols=[1, 2], dtype={\"target\": np.bool})\n    df_test = pd.read_csv(PATH_TEST, index_col=0)\n    \n    # Text preprocessing\n    print(\"Preprocessing and fitting tokenizers\")\n    preprocess(df_train)\n    preprocess(df_test)\n    \n    # Fit tokenizers\n    tokenizer = fit_tokenizer(df_train, df_test, max_num_words=MAX_NUM_WORDS)\n    \n    # Load embedding\n    print(\"Loading embedding\")\n    embedding_weights = EmbeddingLoader.load(\"glove\", tokenizer)\n\n    # Compute model input and build model\n    print(\"Computing input for model training\")\n    input_train = compute_model_input(df_train, tokenizer)\n    target_train = df_train[\"target\"]\n    del df_train\n    gc.collect()\n    \n    if tune_params:\n        print(\"Tuning hyperparameters\")\n        # Hyperparameter search space\n        hp_space = {\n            \"learning_rate\": hp.loguniform(\"learning_rate\", -8, 0),\n            \"batch_size\": hp.choice(\"batch_size\", [64, 256, 1024]),\n            \"early_stopping\": hp.randint(\"early_stopping\", 2),\n            \"neg_class_weight\": hp.uniform(\"neg_class_weight\", 0, 1),\n            \"pos_threshold\": hp.uniform(\"pos_threshold\", 0.2, 1)     # Threshold for classification\n        }\n        best_idx = fmin(partial(run_cross_validation, input_train, target_train, embedding_weights),\n                        space=hp_space, algo=tpe.suggest, max_evals=HP_TUNING_EVALS)\n        hparams = space_eval(space, best_idx)\n        print(f\"Best hyperparameters: {hparams}\")\n    else:\n        hparams = {\n            \"learning_rate\": 0.004,\n            \"batch_size\": 1024,\n            \"early_stopping\": 1,\n            \"neg_class_weight\": 0.35,\n            \"pos_threshold\": 0.5\n        }\n    \n    if test_output_path:\n        print(\"Training model\")\n        model = train_model(input_train, target_train, embedding_weights, hparams)\n        model.summary()\n\n        print(\"Classify test set\")\n        input_test = compute_model_input(df_test, tokenizer)\n        result = compute_predictions(model, input_test, hparams)\n        df_result = pd.DataFrame(result, columns=[\"prediction\"], index=df_test.index, dtype=np.int8)\n        df_result.to_csv(test_output_path)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"run(PATH_RESULT, False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}