{"cells":[{"metadata":{"_kg_hide-output":false,"_uuid":"fafa179d7ad8038fa400343ec62bcf14c8804c5a"},"cell_type":"markdown","source":"Right now, it seems no one post a kernel use word2vec embeddings, so I just create this kernel to show how to use gensim to load word2vec embeddings. I would thanks for the [great kernel](https://www.kaggle.com/sudalairajkumar/a-look-at-different-embeddings) shared by SRK"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b0dba0d31dde40b1dcb2b33cd66d3975833e511a"},"cell_type":"markdown","source":"Load modules include gensim"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import time\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers\n\nfrom gensim.models import KeyedVectors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b633557901e55e80bac0562231521e786bfb4fe"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"13832700829d97ac53e7ca966b95d7647cb34727"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d5356c947ca5572d9cdbe4766aeda9763fca3fa1"},"cell_type":"markdown","source":"Key step, we use KeyedVectors to load the binary embedding files"},{"metadata":{"trusted":true,"_uuid":"ebce722d38bbb64b22ad8e6543af15d9388d6f4a"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = (np.random.rand(nb_words, embed_size) - 0.5) / 5.0\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    if word in embeddings_index:\n        embedding_vector = embeddings_index.get_vector(word)\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"25b7659ae4c19dc31e3dca9d19f20c1bc515bb74"},"cell_type":"code","source":"embed_size = 300\ndef get_model():\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n    # x = CuDNNGRU(64, return_sequences=True)(x)\n    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    conc = concatenate([avg_pool, max_pool])\n    conc = Dense(64, activation=\"relu\")(conc)\n    conc = Dropout(0.1)(conc)\n    outp = Dense(1, activation=\"sigmoid\")(conc)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n\n    return model\n\nmodel = get_model()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e3d3018ee3d378b86889693edc02e1d8f83489cd"},"cell_type":"code","source":"epochs=2\nfor e in range(epochs):\n    model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))\n    pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\n    \n    best_thresh = 0.5\n    best_score = 0.0\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        score = metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))\n        if score > best_score:\n            best_thresh = thresh\n            best_score = score\n            \n    print(\"Val F1 Score: {:.4f}\".format(best_score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b2e57b3835b7c4de6abfe749d1ea580099614aa"},"cell_type":"code","source":"pred_glove_test_y = model.predict([test_X], batch_size=1024, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b4b185dfd8f1cdec29273d9c3bf50226ad87004d"},"cell_type":"code","source":"pred_test_y = (pred_glove_test_y>best_thresh).astype(int)\nout_df = pd.DataFrame({\"qid\":test_df[\"qid\"].values})\nout_df['prediction'] = pred_test_y\nout_df.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}