{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import *\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d063c9ced9a02e3b481208da2b9c6c430244c0aa"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9463d10f9298e705c70b470ef46b0ce30ea6272e"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"245bc5842d93b77a3b839ca10758e315ee8209fd"},"cell_type":"code","source":"def softmax(x, axis=1):\n    \"\"\"Softmax activation function.\"\"\"\n    ndim = K.ndim(x)\n    if ndim == 2:\n        return K.softmax(x)\n    elif ndim > 2:\n        e = K.exp(x - K.max(x, axis=axis, keepdims=True))\n        s = K.sum(e, axis=axis, keepdims=True)\n        return e / s\n    else:\n        raise ValueError('Cannot apply softmax to a tensor that is 1D')\n        \n        \n\ndensor1 = Dense(32, activation = \"tanh\")\ndensor2 = Dense(1, activation = \"relu\")\nactivator = Activation(softmax, name='attention_weights') # We are using a custom softmax(axis = 1) loaded in this notebook\ndotor = Dot(axes = 1)\n\ndef one_step_attention(a):\n    e = densor1(a)\n    energies = densor2(e)\n    alphas = activator(energies)\n    context = dotor([alphas,a])\n    return context\n\ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(CuDNNLSTM(32, return_sequences= True))(x)\nx = BatchNormalization()(x)\nx = Dropout(0.25)(x)\nx = Bidirectional(CuDNNLSTM(32, return_sequences= True))(x)\nx = BatchNormalization()(x)\nx = Dropout(0.25)(x)\ncontext = one_step_attention(x)\ncontext = Flatten()(context)\nmerged = BatchNormalization()(context)\nmerged = Dropout(0.25)(merged)\npreds = Dense(1, activation= 'sigmoid')(merged)\nmodel = Model(inputs = [inp], outputs= preds)\nmodel.compile(loss='binary_crossentropy', optimizer= 'rmsprop', metrics= ['accuracy'])\nprint(model.summary())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ffc58f7b655dbc7083093f9a663b3135ed95019"},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"58cf48eded9ccf2304dab14e20e81053ad99731d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"578fce36d4725389422908924993c659616c325f"},"cell_type":"code","source":"pred_noemb_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_noemb_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7d34be0ea786738bd58f7b8e9b57bd5a68f8f71f"},"cell_type":"code","source":"pred_noemb_test_y = model.predict([test_X], batch_size=1024, verbose=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f28b5647122c3b6f5a4830eea902cf57dcd1c028"},"cell_type":"code","source":"del model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"697158012709250f99ba68b4246b8009fbe08500"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9341d76cf653d8affa511d577069a21f9f05d6e1"},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(CuDNNLSTM(32, return_sequences= True))(x)\nx = BatchNormalization()(x)\nx = Dropout(0.25)(x)\nx = Bidirectional(CuDNNLSTM(32, return_sequences= True))(x)\nx = BatchNormalization()(x)\nx = Dropout(0.25)(x)\ncontext = one_step_attention(x)\ncontext = Flatten()(context)\nmerged = BatchNormalization()(context)\nmerged = Dropout(0.25)(merged)\npreds = Dense(1, activation= 'sigmoid')(merged)\nmodel = Model(inputs = [inp], outputs= preds)\nmodel.compile(loss='binary_crossentropy', optimizer= 'rmsprop', metrics= ['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"907feccdd3ace349b53a61b6587de80f6ab556b6"},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=3, validation_data=(val_X, val_y))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3a808a7632a420a9698fe3278480b9d41a6bd36a"},"cell_type":"code","source":"pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f3b98f29605104e826022dcd9121506b0b661bc"},"cell_type":"code","source":"pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bf5e1cb20615288aba2f987e6b551a676d981861"},"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6b6fc60612495d19287af617c5ca189cd20fa728"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/wiki-news-300d-1M/wiki-news-300d-1M.vec'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE) if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"24bd4a8a8d9521db884fb6e82aed8fff44666205"},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(CuDNNLSTM(32, return_sequences= True))(x)\nx = BatchNormalization()(x)\nx = Dropout(0.25)(x)\nx = Bidirectional(CuDNNLSTM(32, return_sequences= True))(x)\nx = BatchNormalization()(x)\nx = Dropout(0.25)(x)\ncontext = one_step_attention(x)\ncontext = Flatten()(context)\nmerged = BatchNormalization()(context)\nmerged = Dropout(0.25)(merged)\npreds = Dense(1, activation= 'sigmoid')(merged)\nmodel = Model(inputs = [inp], outputs= preds)\nmodel.compile(loss='binary_crossentropy', optimizer= 'rmsprop', metrics= ['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe87e316459f0bfe95f93ac6883cd20d58e048d9"},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0caf9f182eddce65f6484d9f473e1117a7fbb7db"},"cell_type":"code","source":"pred_fasttext_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_fasttext_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"42ff4fcf16e50d7a63a842f5c1cf2b921eb1a802"},"cell_type":"code","source":"pred_fasttext_test_y = model.predict([test_X], batch_size=1024, verbose=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e8e6018d1048beed28c625a6d9d99a5aa602e002"},"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0278d694fafba8ee6743f13e23349eeeca01be44"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE, encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe97bad6d6cfd880094b2dbe64538fe76fe58484"},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(CuDNNLSTM(32, return_sequences= True))(x)\nx = BatchNormalization()(x)\nx = Dropout(0.25)(x)\nx = Bidirectional(CuDNNLSTM(32, return_sequences= True))(x)\nx = BatchNormalization()(x)\nx = Dropout(0.25)(x)\ncontext = one_step_attention(x)\ncontext = Flatten()(context)\nmerged = BatchNormalization()(context)\nmerged = Dropout(0.25)(merged)\npreds = Dense(1, activation= 'sigmoid')(merged)\nmodel = Model(inputs = [inp], outputs= preds)\nmodel.compile(loss='binary_crossentropy', optimizer= 'rmsprop', metrics= ['accuracy'])\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"83b96b1d51118bf7f74ac1799cc46149b88e953a"},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=2, validation_data=(val_X, val_y))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"083a4dd21ff24ac3242147ccbb3b2036cfc1c705"},"cell_type":"code","source":"pred_paragram_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_paragram_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bd21274767f6fac990f86185a2cb9e604da9024e"},"cell_type":"code","source":"pred_paragram_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aac5b3e7458ebe8f63e18747611e40de0c483aec"},"cell_type":"code","source":"del word_index, embeddings_index, all_embs, embedding_matrix, model, inp, x\nimport gc; gc.collect()\ntime.sleep(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ff87a88f8e7335e478063bc79a745a1e02b1be9d"},"cell_type":"code","source":"pred_val_y = 0.33*pred_glove_val_y + 0.33*pred_fasttext_val_y + 0.34*pred_paragram_val_y \nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_val_y>thresh).astype(int))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"05d5f0cff4d93c59b4f2ff6edb7b412a06b0cc36"},"cell_type":"code","source":"pred_test_y = 0.33*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.34*pred_paragram_test_y\npred_test_y = (pred_test_y>0.35).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6e4467d44871fc7ad5d978609c8f701b9b2849d7"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"82c9b75e836154ce3685fd4982e8501ecc714e00"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fe27fc1a34108b4dbe4d813d016260c7f7418538"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}