{"cells":[{"metadata":{"trusted":true,"_uuid":"1d31dea80923ece734de11ad1d61061ca3501010"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPool1D\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\nnp.random.seed(1337)  # for reproducibility\nfrom keras.models import *\nfrom keras.layers import Input, Dense, merge, Permute, Reshape\nimport tensorflow as tf\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"520415a45b9ce8f166eb0c161188067a341a36ff"},"cell_type":"code","source":"INPUT_DIM = 2\nTIME_STEPS = 100\n# if True, the attention vector is shared across the input_dimensions where the attention is applied.\nSINGLE_ATTENTION_VECTOR = False\nAPPLY_ATTENTION_BEFORE_LSTM = False\n\ndef attention_3d_block(inputs):\n    # inputs.shape = (batch_size, time_steps, input_dim)\n    input_dim = int(inputs.shape[2])\n    a = Permute((2, 1))(inputs)\n    a = Dense(TIME_STEPS, activation='softmax')(a)\n    if SINGLE_ATTENTION_VECTOR:\n        a = Lambda(lambda x: K.mean(x, axis=1), name='dim_reduction')(a)\n        a = RepeatVector(input_dim)(a)\n    a_probs = Permute((2, 1), name='attention_vec')(a)\n    output_attention_mul = merge.multiply([inputs, a_probs], name='attention_mul')\n    return output_attention_mul\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9d504b5aa60e1520f6c7f9cc5b9743d83888dc80"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)\n\n\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"229612f2115bf836b29e60318160db1353021d12"},"cell_type":"code","source":"train_df['target'].value_counts()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4e0146e9042dfb83a57495e16416bd176ec4457"},"cell_type":"code","source":"#[print(x) for x in train_df[['question_text']][:10].values]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09ffd2f43c684ed968cd2f985e87ab7ddf8deac1"},"cell_type":"code","source":"train_df['target'].value_counts()\n#train_df_0 = train_df[train_df['target']==0].sample(80810)\ntrain_df_1 = train_df[train_df['target']==1].sample(80810)\ntrain_df = train_df.append([train_df_1]*12)\nprint(train_df.shape)\nprint(train_df['target'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"98061b1eae848c136727063e775f13dadaf807a2"},"cell_type":"code","source":"#train_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1db982e1e965cb8fb5e8c30cdec9f48b92ea0b01"},"cell_type":"code","source":"\n\n## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values\n\n\n\n\n#Without Pretrained Embeddings Withour attention\ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size)(inp)\nx = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\nprint(model.summary())\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true,"_kg_hide-output":false,"_kg_hide-input":false},"cell_type":"code","source":"#model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ec19737aa8c75266f762c5bbbe942da1a41c813"},"cell_type":"code","source":"\"\"\"without_embedding_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (without_embedding_val_y>thresh).astype(int))))\n\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25614d4f3f01440e10e97efdecee2aed58efee81"},"cell_type":"code","source":"\"\"\"#with embedding but without attention\nEMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        \ninp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(16, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8cf449b0f3bb6a647140330b6048651891f29cf"},"cell_type":"code","source":"#model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfcc023ed39f4c208be8b2ba85412b01826ed4b9"},"cell_type":"code","source":"\"\"\"pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))))\n\n\"\"\"\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"683dd615dfad9d60502ad68b8d0125d63757853e"},"cell_type":"code","source":"\n\n#with embedding and with attention\nprint(\"code started...\")\nEMBEDDING_FILE ='../input/embeddings/paragram_300_sl999/paragram_300_sl999.txt' #'../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\n\nprint(\"Extracting the vecs....\")\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE,encoding=\"utf8\", errors='ignore') if len(o)>100)\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n\nprint(\"Creating embedding matrix...\")\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: embedding_matrix[i] = embedding_vector\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"8f07ff871a6e6235d02db4a0e2764ef15c71326d"},"cell_type":"code","source":"#embedding_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f19f45f7ee047e4a4bc8807d73246802c5276074"},"cell_type":"code","source":"#np.savetxt('embedding_matrix.csv',embedding_matrix)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"19c7a99d137833d77af83c88747307e730086bdc"},"cell_type":"code","source":"inp = Input(shape=(maxlen,))\nx = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\nx = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\nx = attention_3d_block(x)\nx = GlobalMaxPool1D()(x)\nx = Dense(40, activation=\"relu\")(x)\nx = Dropout(0.1)(x)\nx = Dense(1, activation=\"sigmoid\")(x)\nmodel = Model(inputs=inp, outputs=x)\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\nprint(model.summary())\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fa763e18ed5f3ae65b4ee8680d7eda8b539b2c4"},"cell_type":"code","source":"model.fit(train_X, train_y, batch_size=512, epochs=5, validation_data=(val_X, val_y))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c999e6c7f47145dced285f5b36b8c105f9de2317"},"cell_type":"code","source":"with_attention_val_y = model.predict([val_X], batch_size=1024, verbose=1)\nfor thresh in np.arange(0.1, 0.501, 0.01):\n    thresh = np.round(thresh, 2)\n    print(\"F1 score at threshold {0} is {1}\".format(thresh, metrics.f1_score(val_y, (with_attention_val_y>thresh).astype(int))))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"18d9aaff2b5523e0bf2294713593804bf26d1a2b"},"cell_type":"code","source":"#pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)\npred_test_y = model.predict([test_X], batch_size=1024, verbose=1)\n#pred_test_y = 0.33*pred_glove_test_y + 0.33*pred_fasttext_test_y + 0.34*pred_paragram_test_y\npred_test_y = (pred_test_y>0.5).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}