{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nfrom sklearn.model_selection import GridSearchCV, StratifiedKFold\nfrom sklearn.metrics import f1_score, roc_auc_score\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.layers import concatenate\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, CuDNNLSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D, GlobalMaxPooling1D, GlobalAveragePooling1D\nfrom keras.layers import Input, Embedding, Dense, Conv2D, MaxPool2D, concatenate\nfrom keras.layers import Reshape, Flatten, Concatenate, Dropout, SpatialDropout1D\nfrom keras.optimizers import Adam\nfrom keras.models import Model\nfrom keras import backend as K\nfrom keras.engine.topology import Layer\nfrom keras import initializers, regularizers, constraints, optimizers, layers# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"df=pd.read_csv('../input/train.csv')\ndftest=pd.read_csv('../input/test.csv')\nprint(df.shape, dftest.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c932f69fbbfb0c949fbaeebc11175d2f49a6854e"},"cell_type":"code","source":"train_df, val_df = train_test_split(df, test_size=0.1, random_state=1995)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"40b72d219f8ff549b80bab26674276bb3daed17c"},"cell_type":"code","source":"train_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = dftest[\"question_text\"].fillna(\"_na_\").values\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e40cc3944bfd33232c9691368ff89c8a1a16e5af"},"cell_type":"code","source":"embed_size = 300 # how big is each word vector\nmax_features = 95000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 70 # max number of words in a question to use","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b50ca86ca8f90ac6a95a4e9dd430d6122523b56c"},"cell_type":"code","source":"#df.astype(int64)\n#df.describe().apply(lambda x: format(x,  '%.3f' % x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4313b8e6ceb49fbdbb73e0b6aedb2335d0e0a781"},"cell_type":"code","source":"tokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"4510133b986857232285e91fe9a37c97c565295e"},"cell_type":"code","source":"train_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ab2b51fef0c506623e3e2daa510016fa66dc0adb"},"cell_type":"code","source":"train_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3dc01ef728693ebda9102a5f1b27e0bdd6800169"},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"850a910aca18e2060beb4b1857d6d6a4ed33e5f8"},"cell_type":"code","source":"vectorizer = TfidfVectorizer()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d09bf4ebfdd5af877222563aadb9e86a4578f438"},"cell_type":"code","source":"train_X.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f01dc2ea72c21481f114f954e896125cf83b182"},"cell_type":"code","source":"def attention_3d_block(inputs):\n    # inputs.shape = (batch_size, time_steps, input_dim)\n    TIME_STEPS = inputs.shape[1].value\n    SINGLE_ATTENTION_VECTOR = False\n    \n    input_dim = int(inputs.shape[2])\n    a = Permute((2, 1))(inputs)\n    a = Reshape((input_dim, TIME_STEPS))(a) # this line is not useful. It's just to know which dimension is what.\n    a = Dense(TIME_STEPS, activation='softmax')(a)\n    if SINGLE_ATTENTION_VECTOR:\n        a = Lambda(lambda x: K.mean(x, axis=1))(a)\n        a = RepeatVector(input_dim)(a)\n    a_probs = Permute((2, 1))(a)\n    output_attention_mul = Multiply()([inputs, a_probs])\n    return output_attention_mul","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f712e729f9b45c7b26582528417d8e0191624a10"},"cell_type":"code","source":"from keras import backend as K\nfrom keras.engine.topology import Layer, InputSpec\nfrom keras import initializers\n\nclass AttLayer(Layer):\n    def __init__(self, attention_dim):\n        self.init = initializers.get('normal')\n        self.supports_masking = True\n        self.attention_dim = attention_dim\n        super(AttLayer, self).__init__()\n\n    def build(self, input_shape):\n        assert len(input_shape) == 3\n        self.W = K.variable(self.init((input_shape[-1], self.attention_dim)))\n        self.b = K.variable(self.init((self.attention_dim, )))\n        self.u = K.variable(self.init((self.attention_dim, 1)))\n        self.trainable_weights = [self.W, self.b, self.u]\n        super(AttLayer, self).build(input_shape)\n\n    def compute_mask(self, inputs, mask=None):\n        return mask\n\n    def call(self, x, mask=None):\n        # size of x :[batch_size, sel_len, attention_dim]\n        # size of u :[batch_size, attention_dim]\n        # uit = tanh(xW+b)\n        uit = K.tanh(K.bias_add(K.dot(x, self.W), self.b))\n        ait = K.dot(uit, self.u)\n        ait = K.squeeze(ait, -1)\n\n        ait = K.exp(ait)\n\n        if mask is not None:\n            # Cast the mask to floatX to avoid float64 upcasting in theano\n            ait *= K.cast(mask, K.floatx())\n        ait /= K.cast(K.sum(ait, axis=1, keepdims=True) + K.epsilon(), K.floatx())\n        ait = K.expand_dims(ait)\n        weighted_input = x * ait\n        output = K.sum(weighted_input, axis=1)\n\n        return output\n\n    def compute_output_shape(self, input_shape):\n        return (input_shape[0], input_shape[-1])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b07775f864b1c9bb5299dea7dc329181b49255fb"},"cell_type":"code","source":"import gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74763ed0f32fee03f20bb0f8067893c6bf0253d7"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix_1 = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix_1[i] = embedding_vector\n\ndel embeddings_index; gc.collect() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8ab080cd6c915636f159f4ded59244fc9672d81"},"cell_type":"code","source":"def model2():\n    inp = Input(shape=(maxlen, ))\n    embed = Embedding(max_features, embed_size, weights=[embedding_matrix_1], trainable=False)(inp)\n    x = embed\n    \n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x = attention_3d_block(x)\n    x = Bidirectional(CuDNNLSTM(128, return_sequences=True))(x)\n    x = AttLayer(64)(x)\n    x = Dropout(0.3)(x)\n    x = Dense(128, activation='relu')(x)\n    outp = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])    \n\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5e9be02944863ae485843c1949d29081f5773ba8"},"cell_type":"code","source":"from keras.models import Model\nfrom keras.layers import Input, Dense, Embedding, concatenate\nfrom keras.layers import CuDNNGRU, Bidirectional, GlobalAveragePooling1D, GlobalMaxPooling1D, Conv1D\nfrom keras.layers import Add, BatchNormalization, Activation, CuDNNLSTM, Dropout\nfrom keras.layers import *\nfrom keras.models import *\nfrom keras.preprocessing import text, sequence\nfrom keras.callbacks import EarlyStopping, ModelCheckpoint, ReduceLROnPlateau\nimport gc\nfrom sklearn import metrics","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3786397741626dc42a3f201ae8b8e247c0949a3d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0690a5a384e2e19bf403a3b031b04422eddfaa71"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c09ae8984d6a9c3f3d8d75ec971f0572d581c8e8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bdde69d00a58a0f34055f6e5f6bbd95bd9c53428"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4837626bbb5e4fcfc27efa2b8339356b72be1121"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e51bce10f66f5b7d7179dd6c6ac0079c87f0146"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"476e51e448f7dbcc4ca6afc8a31c5e5958d13894"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"50dbdf68769e0fcf9f94d65ad71444943058867a"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6a88dd799c1de5dded23dc1544c1aabf38b666fe"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"749db5d74a1ba2454a1d474cbf215d8b5ee847d4"},"cell_type":"code","source":"F1scores_epochs = [0.6327, 0.6607, 0.6688]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cf35ee060b7b34cd1451421a0051223e4de09c2f"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"33e2333a25eaf066b0f05db16fd859a4c2eb3d63"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e732eae36ed6e77ef6610f08f3a83c57a8a1f242"},"cell_type":"code","source":"F1scores_epochs = [0.6327, 0.6607, 0.6688, 0.6698]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"02dca9a29694c2ce134bf3cd074999e07d7ace5d","scrolled":true},"cell_type":"code","source":"MODEL2 = model2()\nMODEL2.summary()\n\nbatch_size = 2048\nepochs = 5\n\nearly_stopping = EarlyStopping(patience=3, verbose=1, monitor='val_loss', mode='min')\nmodel_checkpoint = ModelCheckpoint('./model2.model', save_best_only=True, verbose=1, monitor='val_loss', mode='min')\nreduce_lr = ReduceLROnPlateau(factor=0.5, patience=3, min_lr=0.0001, verbose=1)\n\nhist = MODEL2.fit(train_X, train_y, batch_size=batch_size, epochs=epochs, validation_data=(val_X, val_y), verbose=True)\nMODEL2.save('./model2.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"be68cdf14e80c863f98fb49e6ba4dd277e7de939"},"cell_type":"code","source":"pred_val_y_2 = MODEL2.predict([val_X], batch_size=1024, verbose=1)\nthresholds = []\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    res = metrics.f1_score(val_y, (pred_val_y_2 > thresh).astype(int))\n    thresholds.append([thresh, res])\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, res))\n    \nthresholds.sort(key=lambda x: x[1], reverse=True)\nbest_thresh_2 = thresholds[0][0]\nprint(\"Best threshold: \", best_thresh_2)\n\ny_pred_2 = MODEL2.predict(test_X, batch_size=1024, verbose=True)\nprint(y_pred_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f0b0d59a96fbd17ee5c691d04a95f337e6086f35"},"cell_type":"code","source":"import matplotlib.pyplot as plt\n%matplotlib inline\nOrder = [1, 2, 3, 4]\nF1scores_epochs = [0.6327, 0.6607, 0.6688, 0.6698]\n\nLABELS = [\"1\", \"2\",\"3\", \"5\"]\nplt.figure(figsize = (10,5))\nplt.bar(Order, F1scores_epochs, align='center', width=0.3)\nplt.xticks(Order, LABELS)\nplt.xlabel('Epochs')\nplt.ylabel('F-1 scores')\nplt.ylim(0.63, 0.675)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9afc95ea0aec569770119844bd1f163a24c15851"},"cell_type":"code","source":"Order = [1, 2, 3, 4]\nValidation_Accuracy = [0.9546, 0.9572, 0.957, 0.6698]\n\nLABELS = [\"1\", \"2\",\"3\", \"5\"]\nplt.figure(figsize = (10,5))\nplt.bar(Order, F1scores_epochs, align='center', width=0.3)\nplt.xticks(Order, LABELS)\nplt.xlabel('Epochs')\nplt.ylabel('F-1 scores')\nplt.ylim(0.63, 0.675)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8d6bc2828a70180105e77150a75e4eed360ead9"},"cell_type":"code","source":"y_te = (y_pred_2[:,0] > best_thresh_2).astype(np.int)\n\nsubmit_df = pd.DataFrame({\"qid\": dftest[\"qid\"], \"prediction\": y_te})\nsubmit_df.to_csv(\"submission.csv\", index=False)\n#sub = pd.read_csv('../input/sample_submission.csv')\n#sub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a7910d3c3b6357fa2b7d84c4e01f60349bf9b9f4"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b2ede65e0bb8ad2906787651a89e8f1a43f33127"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}