{"cells":[{"metadata":{"trusted":true,"_uuid":"210328d08dd545db458f30549ca43002f161f00a"},"cell_type":"code","source":"import tensorflow as tf\nwith tf.Session() as sess:\n    devices = sess.list_devices()\ndevices","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport gc\nimport time\nfrom tqdm import tqdm\nimport math\nimport pickle\n\nfrom sklearn.model_selection import train_test_split\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nfrom keras import backend as K\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint\nfrom sklearn import metrics\n\nclass Attention(Layer):\n    def __init__(self, step_dim,\n                 W_regularizer=None, b_regularizer=None,\n                 W_constraint=None, b_constraint=None,\n                 bias=True, **kwargs):\n        self.supports_masking = True\n        self.init = initializers.get('glorot_uniform')\n\n        self.W_regularizer = regularizers.get(W_regularizer)\n        self.b_regularizer = regularizers.get(b_regularizer)\n\n        self.W_constraint = constraints.get(W_constraint)\n        self.b_constraint = constraints.get(b_constraint)\n\n        self.bias = bias\n        self.step_dim = step_dim\n        self.features_dim = 0\n        super(Attention, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight((input_shape[-1],),\n                                 initializer=self.init,\n                                 name='{}_W'.format(self.name),\n                                 regularizer=self.W_regularizer,\n                                 constraint=self.W_constraint)\n        self.features_dim = input_shape[-1]\n\n        if self.bias:\n            self.b = self.add_weight((input_shape[1],),\n                                     initializer='zero',\n                                     name='{}_b'.format(self.name),\n                                     regularizer=self.b_regularizer,\n                                     constraint=self.b_constraint)\n        else:\n            self.b = None\n\n        self.built = True\n\n\n    def compute_mask(self, input, input_mask=None):\n        return None\n\n    def call(self, x, mask=None):\n        features_dim = self.features_dim\n        step_dim = self.step_dim\n\n        eij = K.reshape(K.dot(K.reshape(x, (-1, features_dim)),\n                        K.reshape(self.W, (features_dim, 1))), (-1, step_dim))\n\n        if self.bias:\n            eij += self.b\n\n        eij = K.tanh(eij)\n\n        a = K.exp(eij)\n\n        if mask is not None:\n            a *= K.cast(mask, K.floatx())\n\n        a /= K.cast(K.sum(a, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n\n        a = K.expand_dims(a)\n        weighted_input = x * a\n        return K.sum(weighted_input, axis=1)\n\n    def compute_output_shape(self, input_shape):\n        return input_shape[0],  self.features_dim\n\ndef load_and_prec(maxlen=300):\n    train_df = pd.read_csv(\"../input/train.csv\")\n    test_df = pd.read_csv(\"../input/test.csv\")\n    print(\"Train shape : \",train_df.shape)\n    print(\"Test shape : \",test_df.shape)\n    \n    ## split to train and val\n    train_df, val_df = train_test_split(train_df, test_size=0.2, random_state=2018)\n\n    ## create training, validation, and test sets\n    print('Creating train, val, and test sets')\n    train_X = train_df[\"question_text\"].values\n    val_X = val_df[\"question_text\"].values   \n    test_X = test_df[\"question_text\"].values\n\n    ## Tokenize the sentences\n    print('Tokenizing the sentences')\n    tokenizer = Tokenizer()\n    tokenizer.fit_on_texts(list(train_X)) \n    train_X = tokenizer.texts_to_sequences(train_X)\n    val_X = tokenizer.texts_to_sequences(val_X)\n    test_X = tokenizer.texts_to_sequences(test_X)\n\n    ## Pad the sentences \n    print('Padding Sequences for input datasets')\n    train_X = pad_sequences(train_X, maxlen=maxlen, padding='pre')\n    val_X = pad_sequences(val_X, maxlen=maxlen, padding='pre')\n    test_X = pad_sequences(test_X, maxlen=maxlen, padding='pre')\n\n    ## Get the target values\n    train_y = train_df['target'].values\n    val_y = val_df['target'].values  \n    \n    ## Shuffling the data\n    print('Shuffling training and validation sets')\n    np.random.seed(2018)\n    trn_idx = np.random.permutation(len(train_X))\n    val_idx = np.random.permutation(len(val_X))\n    train_X = train_X[trn_idx]\n    val_X = val_X[val_idx]\n    train_y = train_y[trn_idx]\n    val_y = val_y[val_idx]    \n    \n    print('Complete - Returning Values')\n    return train_X, val_X, test_X, train_y, val_y, tokenizer.word_index\n\n\ndef load_all_embeddings(vocabulary,max_features,embed_size):\n    \n    def load_embedding(embed_file):\n        def get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n        if 'glove' in embed_file:\n                embed_file = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\n                embeddings_index = dict(get_coefs(*o.split(\" \")) for o\\\n                                        in open(embed_file))\n        if 'wiki' in embed_file:\n                embed_file = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\n                embeddings_index = dict(get_coefs(*o.split(\" \")) for o\\\n                                        in open(embed_file) if len(o)>100)\n        if 'paragram' in embed_file:\n                embed_file = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\n                embeddings_index = dict(get_coefs(*o.split(\" \")) for o\\\n                                        in open(embed_file, encoding=\"utf8\", errors='ignore')\\\n                                        if len(o)>100)\n        if 'Google' in embed_file:\n                from gensim.models import KeyedVectors\n                embed_file = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n                embeddings_index = KeyedVectors.load_word2vec_format(embed_file, binary=True)\n                nb_words = min(max_features, len(vocabulary))\n                embedding_matrix = (np.random.rand(nb_words, embed_size) - 0.5) / 5.0\n                for word, i in vocabulary.items():\n                    if i >= max_features: continue\n                    if word in embeddings_index:\n                        embedding_vector = embeddings_index.get_vector(word)\n                        embedding_matrix[i] = embedding_vector\n                \n                del embeddings_index; gc.collect()\n                return embedding_matrix\n\n        ## Read Embeddings for Specified File       \n        all_embs = np.stack(embeddings_index.values())\n        emb_mean,emb_std = all_embs.mean(), all_embs.std()\n        nb_words = min(max_features, len(vocabulary))\n\n        ## Initialize embedding matrix with random values based on -\n        ## the mean and std (Not all words are contained in every pre-train)\n        embedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n        for word, i in vocabulary.items():\n            if i >= max_features: continue\n            embedding_vector = embeddings_index.get(word)\n            if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \n        del embeddings_index; gc.collect()\n        return embedding_matrix\n    \n    #Loop over embeddings and create output file\n    embed_list = os.listdir('../input/embeddings/')\n    embedding_matrices = []\n    for embed in embed_list:\n        embedding_matrices.append(load_embedding(embed))\n        print('Loaded %s' % embed)\n    #Average embedding matrice\n    average_embedding_matrix = np.mean(embedding_matrices,axis=0)\n    print(np.shape(average_embedding_matrix))\n    \n    return average_embedding_matrix","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fa5e1bdefddb4c197f9bc5499ff086337d21209c"},"cell_type":"code","source":"## Models\ndef joined_model(embedding_matrix, maxlen, max_features,embed_size):\n    inp = Input(shape=(maxlen,))\n    emb = Embedding(max_features, embed_size, weights=[embedding_matrix], trainable=False)(inp)\n    \n    #BDLSTM-att w/Dropout\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(emb)\n    x = Dropout(0.2)(x)\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x = Dropout(0.2)(x)\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x = Attention(maxlen)(x)\n    x = Dense(128, activation=\"relu\")(x)\n    \n    ## Simple BDLSTM-att\n    y = Bidirectional(CuDNNLSTM(128, return_sequences=True))(emb)\n    y = Bidirectional(CuDNNLSTM(128, return_sequences=True))(y)\n    y = Attention(maxlen)(y)\n    \n    #Simple BDLSTM-maxpool\n    z = Bidirectional(CuDNNLSTM(128, return_sequences=True))(emb)\n    z = Bidirectional(CuDNNLSTM(128, return_sequences=True))(z)\n    z = GlobalMaxPool1D()(z)\n    z = Dense(256, activation=\"sigmoid\")(z)\n    \n    #LSTM_du\n    w = Bidirectional(CuDNNGRU(64, return_sequences=True))(emb)\n    avg_pool = GlobalAveragePooling1D()(w)\n    max_pool = GlobalMaxPooling1D()(w)\n    w_conc = concatenate([avg_pool, max_pool])\n    w_conc = Dense(64, activation=\"relu\")(w_conc)\n    w_conc = Dropout(0.1)(w_conc)\n\n    #Concatenate\n    output = concatenate([x,y,z,w_conc])\n    output = Dense(1, activation=\"sigmoid\")(output)\n    model = Model(inputs=inp, outputs=output)\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    return model\n\ndef train_pred(model, train_X, train_y, val_X, val_y, test_X, epochs=2):\n    file_path = \"model-checkpoint.hdf5\"\n    checkpoint = ModelCheckpoint(file_path, monitor='val_loss', verbose=1,save_best_only=True, mode='min')\n    early_stop = EarlyStopping(monitor=\"val_loss\", mode=\"min\", patience=1)\n\n    model.fit(\n        train_X, \n        train_y, \n        batch_size=1024, \n        epochs=epochs, \n        validation_data=(val_X, val_y),\n        callbacks=[checkpoint,early_stop]\n    )\n    \n    model.load_weights(file_path)\n    pred_val_y = model.predict([val_X], batch_size=256, verbose=0)\n    best_thresh = 0.5\n    best_score = 0.0\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        score = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n        if score > best_score:\n            best_thresh = thresh\n            best_score = score\n\n    print(\"Val F1 Score: {:.4f}\".format(best_score))\n\n    pred_test_y = model.predict([test_X], batch_size=1024, verbose=0)\n    return pred_val_y, pred_test_y, best_score","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1fb8a5f1197215e635594f66ad8269410e687288"},"cell_type":"markdown","source":"## Get Model Data and Settings"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"## Text Input Settings\nembed_size = 300 # how big is each word vector\nmax_features = 75000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 65 # max number of words in a question to use\n\n#Get training data\ntrain_X, val_X, test_X, train_y, val_y, vocabulary =\\\nload_and_prec(maxlen)\n\n#Get Embedding Matrix for Model\nembed_matrix = load_all_embeddings(vocabulary,max_features,embed_size)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a02700b7b1119dfedfefa2fe8297797997a86c4c"},"cell_type":"markdown","source":"## Set Models, Fit, and Generate Predictions"},{"metadata":{"trusted":true,"_uuid":"bd2b45b149afa28139a0b3b102429c005e6053b6"},"cell_type":"code","source":"model = joined_model(embed_matrix, maxlen, max_features, embed_size)\n\npred_val_y, pred_test_y, best_score =\\\ntrain_pred(model, train_X, train_y, val_X, val_y, test_X, epochs = 8)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"749a5cb978ac68a6a926fd051eab834927dcbcef"},"cell_type":"markdown","source":"## Find Best Thresholds and Generate Scores"},{"metadata":{"trusted":true,"_uuid":"930677cc86c508de343cce1571cc6ff18699967e"},"cell_type":"code","source":"thresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(val_y, (pred_val_y > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8eb86d888fe77771e9fbc782c43cc9c8bf6c84d6"},"cell_type":"markdown","source":"## Output Predictions"},{"metadata":{"trusted":true,"_uuid":"34f2b50ceed908ecedfb9fdc9cdd902aae0e5b08"},"cell_type":"code","source":"pred_test_y = (pred_test_y > best_thresh).astype(int)\ntest_df = pd.read_csv(\"../input/test.csv\", usecols=[\"qid\"])\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2a5e27d5037656d9466bc2fcaa7b75c997f67d5d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}