{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\nimport tensorflow as tf\n\nSEED = 2018\n\nnp.random.seed(SEED)\ntf.set_random_seed(SEED)\n\nfrom tqdm import tqdm\ntqdm.pandas()\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import *\nfrom tensorflow.keras.models import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\n\nfrom tensorflow.keras import backend as K\n#from keras.engine.topology import Layer\nfrom tensorflow.keras.layers import Layer\nfrom tensorflow.keras import initializers, regularizers, constraints, optimizers, layers\n\nimport gc, re\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score\n#from sklearn.model_selection import train_test_split\n\n# from nltk.corpus import stopwords\n# sw = stopwords.words(\"english\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"90b5e8186e3c5dfaa16dbce540331c49dcc11801"},"cell_type":"code","source":"# def remove_stopwords(x):\n#     x = str(x)\n#     return ' '.join([w for w in x.split(' ') if w not in sw])\n\ndef clean_text(x):\n    x = str(x)\n    for punct in \"/-'\":\n        x = x.replace(punct, ' ')\n    for punct in '&':\n        x = x.replace(punct, f' {punct} ')\n    for punct in '?!.,\"#$%\\'()*+-/:;<=>@[\\\\]^_`{|}~' + '“”’':\n        x = x.replace(punct, '')\n    return x\n\ndef clean_numbers(x):\n    x = re.sub('[0-9]{5,}', '#####', x)\n    x = re.sub('[0-9]{4}', '####', x)\n    x = re.sub('[0-9]{3}', '###', x)\n    x = re.sub('[0-9]{2}', '##', x)\n    return x\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\nmispell_dict = {'colour':'color',\n                'centre':'center',\n                'didnt':'did not',\n                'doesnt':'does not',\n                'isnt':'is not',\n                'shouldnt':'should not',\n                'favourite':'favorite',\n                'travelling':'traveling',\n                'counselling':'counseling',\n                'theatre':'theater',\n                'cancelled':'canceled',\n                'labour':'labor',\n                'organisation':'organization',\n                'wwii':'world war 2',\n                'citicise':'criticize',\n                'instagram': 'social medium',\n                'whatsapp': 'social medium',\n                'snapchat': 'social medium'\n                }\nmispellings, mispellings_re = _get_mispell(mispell_dict)\n\ndef replace_typical_misspell(text):\n    def replace(match):\n        return mispellings[match.group(0)]\n    return mispellings_re.sub(replace, text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"417fc7b1456a0340dbbf7b5afe44432ed7a0f5d9"},"cell_type":"code","source":"def load_and_preprocess_data(max_features=50000, maxlen=70):\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n    # Remove stoping words\n    #train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: remove_stopwords(x))\n    #test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: remove_stopwords(x))\n    \n    # Clean the text\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_text(x))\n    \n    # Clean numbers\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: clean_numbers(x))\n    \n    # Clean speelings\n    train_df[\"question_text\"] = train_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    test_df[\"question_text\"] = test_df[\"question_text\"].progress_apply(lambda x: replace_typical_misspell(x))\n    \n    ## fill up the missing values\n    train_X = train_df[\"question_text\"].fillna(\"_##_\").values\n    test_X = test_df[\"question_text\"].fillna(\"_##_\").values\n\n    ## Tokenize the sentences\n    tokenizer = Tokenizer(num_words=max_features)\n    tokenizer.fit_on_texts(list(train_X))\n    train_X = tokenizer.texts_to_sequences(train_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    train_X = pad_sequences(train_X, maxlen=maxlen)\n    test_X = pad_sequences(test_X, maxlen=maxlen)\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    \n    #shuffling the data\n    np.random.seed(2018)\n    trn_idx = np.random.permutation(len(train_X))\n\n    train_X = train_X[trn_idx]\n    train_y = train_y[trn_idx]\n    \n    return train_X, test_X, train_y, tokenizer.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"63e672daf4c1baa189fb60d0585ddd55da210fc8"},"cell_type":"code","source":"def load_glove(word_index):\n    EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n            \n    return embedding_matrix \n    \ndef load_fasttext(word_index):    \n    EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n\n    return embedding_matrix\n\ndef load_para(word_index):\n    EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n    def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n    embeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\n    all_embs = np.stack(embeddings_index.values())\n    emb_mean,emb_std = all_embs.mean(), all_embs.std()\n    embed_size = all_embs.shape[1]\n\n    # word_index = tokenizer.word_index\n    nb_words = min(max_features, len(word_index))\n    embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n    for word, i in word_index.items():\n        if i >= max_features: continue\n        embedding_vector = embeddings_index.get(word)\n        if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n    \n    return embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2aa88090981b6310b6644a0f3acc6dbd2b3d9741"},"cell_type":"code","source":"class Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight('{}_W'.format(self.name), shape=(input_shape[-1].value,),\n                                 initializer=self.init,\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight('{}_b'.format(self.name), shape=(input_shape[1].value,),\n                                     initializer='zero',\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint\n                                    )\n        else:\n            self.b = None\n\n        self.built = True\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"79fcb1c003aca5f86fa0dbb10ce1d08c6506231f"},"cell_type":"code","source":"def model_lstm_attention(embedding_matrix, maxlen=70, max_features=50000, units=64):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNLSTM(units*2, return_sequences=True))(x)\n    x = Bidirectional(CuDNNLSTM(units, return_sequences=True))(x)\n    x = Attention(maxlen)(x)\n    x = Dense(units, activation='relu')(x)\n    outp= Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])\n    return model\n\ndef model_gru_attention(embedding_matrix, maxlen=70, max_features=50000, units=64):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNGRU(units*2, return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(units, return_sequences=True))(x)\n    x = Attention(maxlen)(x)\n    x = Dense(units, activation='relu')(x)\n    outp= Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])\n    return model\n\ndef model_gru_attention_3(embedding_matrix, maxlen=70, max_features=50000, units=64):\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    x = Bidirectional(CuDNNGRU(units*2, kernel_initializer=initializers.glorot_uniform(seed = SEED), return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(units, kernel_initializer=initializers.glorot_uniform(seed = SEED), return_sequences=True))(x)\n    x = Bidirectional(CuDNNGRU(units//2, kernel_initializer=initializers.glorot_uniform(seed = SEED), return_sequences=True))(x)\n    x = Attention(maxlen)(x)\n    outp= Dense(1, kernel_initializer=initializers.he_uniform(seed=SEED), activation='sigmoid')(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy', optimizer=Adam(), metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"65d008ef0b60ac644eb55cc0fc8a726bf50f5414"},"cell_type":"code","source":"def train_pred(model, train_X, train_y, val_X, val_y, epochs=2):\n    for e in range(epochs):\n        model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))\n        pred_val_y = model.predict([val_X], batch_size=1024, verbose=0)\n\n        best_thresh = 0.5\n        best_score = 0.0\n        for thresh in np.arange(0.1, 0.501, 0.01):\n            thresh = np.round(thresh, 2)\n            score = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n            if score > best_score:\n                best_thresh = thresh\n                best_score = score\n\n        print(\"Val F1 Score: {:.4f}\".format(best_score))\n\n    pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)\n    print('='*100)\n    return pred_val_y, pred_test_y, best_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"16377fced6017ddc0d40e12f71da9f37897161cb"},"cell_type":"code","source":"embed_size = 300\nmax_features = 100000\nmaxlen = 40","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b370506e3512111916c3f7c012f1a6a145e6470","scrolled":true},"cell_type":"code","source":"import time\nstart_time = time.time()\n\ntrain_X, test_X, train_y, word_index = load_and_preprocess_data(max_features=max_features, maxlen=maxlen)\n\ntotal_time = (time.time() - start_time)/60.0\nprint(\"Took {0} minutes\".format(total_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58d920c9d05e253a473099a5a0ee4328f5b3dd71"},"cell_type":"code","source":"import time\nstart_time = time.time()\n\nembedding_matrix_1 = load_glove(word_index)\n#embedding_matrix_2 = load_fasttext(word_index)\nembedding_matrix_3 = load_para(word_index)\n\ntotal_time = (time.time() - start_time)/60.0\nprint(\"Took {0} minutes\".format(total_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"730f4de3b8bc15b84adda90268eaeac44bac28c2"},"cell_type":"code","source":"embedding_matrix = np.mean([embedding_matrix_1, embedding_matrix_3], axis = 0)\nprint(np.shape(embedding_matrix))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dab72672c52e429f4687c033cfd84d4ce7b1c525"},"cell_type":"code","source":"def threshold_search(y_true, y_proba):\n    best_threshold = 0\n    best_score = 0\n    for threshold in [i * 0.01 for i in range(100)]:\n        score = f1_score(y_true=y_true, y_pred=y_proba > threshold)\n        if score > best_score:\n            best_threshold = threshold\n            best_score = score\n    search_result = {'threshold': best_threshold, 'f1': best_score}\n    return search_result","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"020dc129d1aa9013bdf6a3d499819190f3edba3d"},"cell_type":"code","source":"train_meta = np.zeros(train_y.shape)\ntest_meta = np.zeros(test_X.shape[0])\nsplits = list(StratifiedKFold(n_splits=4, shuffle=True, random_state=SEED).split(train_X, train_y))\n\nfor idx, (train_idx, valid_idx) in enumerate(splits):\n        X_train = train_X[train_idx]\n        y_train = train_y[train_idx]\n        X_val = train_X[valid_idx]\n        y_val = train_y[valid_idx]\n        \n        tmp_result = []\n        \n        model = model_gru_attention_3(embedding_matrix, maxlen=maxlen, max_features=max_features, units=64)\n        pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = 2)\n        tmp_result.append([pred_val_y, pred_test_y, best_score])\n        \n        model = model_gru_attention(embedding_matrix, maxlen=maxlen, max_features=max_features, units=64)\n        pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = 2)\n        tmp_result.append([pred_val_y, pred_test_y, best_score])\n        \n        model = model_lstm_attention(embedding_matrix_1, maxlen=maxlen, max_features=max_features, units=64)\n        pred_val_y, pred_test_y, best_score = train_pred(model, X_train, y_train, X_val, y_val, epochs = 2)\n        tmp_result.append([pred_val_y, pred_test_y, best_score])\n        \n        pred_val_y, pred_test_y, best_score = np.mean(tmp_result, axis=0)\n        print(\"Mean Result: \", best_score)\n        \n        train_meta[valid_idx] = pred_val_y.reshape(-1)\n        test_meta += pred_test_y.reshape(-1) / len(splits)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7cced03c9fa80299dd7d2ca67bfe7966b210113b"},"cell_type":"code","source":"search_result = threshold_search(train_y, train_meta)\nprint(search_result)\n\nsub = pd.read_csv('../input/sample_submission.csv')\nsub.prediction = test_meta > search_result['threshold']\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8c7329e65917d8781db18e25b75d10b3eef1be7b"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}