{"cells":[{"metadata":{"_uuid":"c725c5d5fb4d331931d21345ef534ee33c4102a0"},"cell_type":"markdown","source":"Word2vec may not be a good embedding in this competition but I plan to use gensim and word2vec to do data augmentation.\n\nWhat I did: Random choose a portion of the words in the positive sentences and replace them with nearest neigbour in word2vec with gensim library.\n\nRef: \n\nhttps://www.kaggle.com/strideradu/word2vec-and-gensim-go-go-go/notebook"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"301794b4df2c99666af8c0bac8530003f1a8cdf5"},"cell_type":"markdown","source":"**Data augmentation**"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors\nfrom gensim.models import Word2Vec\n\nEMBEDDING_FILE = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\nembeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d819216fbee96a3c8ab645a4b67875e93174348b"},"cell_type":"code","source":"# Randomly replace some words\ndef replace_words(text, ratio):\n    word_list = str(text).split(' ')\n    n = len(word_list)\n    chosen_ind = np.random.choice([i for i in range(n)], int(n * ratio), replace=False) \n    for i in chosen_ind:\n        if word_list[i] in embeddings_index:\n            word_list[i] = embeddings_index.most_similar(word_list[i],topn=1)[0][0]\n    return ' '.join(word_list)\n\nreplace_words('I am a data scientist and I love machine learning.', 1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dd7260c1e37c957d8504c0ad8ad021da7507ddc4"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\n\ntrain_fake = train_df.sample(n=10, replace=True)\ntrain_fake[\"real_question_text\"] = train_fake[\"question_text\"]\ntrain_fake[\"question_text\"] = train_fake[\"question_text\"].apply(lambda x: replace_words(x, 0.3))\n\ntrain_fake.head(n=10) ","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"befac4ad09413b517feee7186a137ad80bc16006"},"cell_type":"markdown","source":"**Original dataset**"},{"metadata":{"trusted":true,"_uuid":"5bfa721ddb32272928643305fd8002d8e0071cc3"},"cell_type":"code","source":"import time\nfrom tqdm import tqdm\nimport math\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.layers import Dense, Input, LSTM, Embedding, Dropout, Activation, CuDNNGRU, Conv1D\nfrom keras.layers import Bidirectional, GlobalMaxPooling1D, GlobalAveragePooling1D, concatenate\nfrom keras.models import Model\nfrom keras import initializers, regularizers, constraints, optimizers, layers","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"452f03a906d4af430603fc3f47379f07732fce9e"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f05ad8f8d1b15ffecea1cf28f756edcc375dfac2"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)\n\n## some config values \nembed_size = 300 # how big is each word vector\nmax_features = 50000 # how many unique words to use (i.e num rows in embedding vector)\nmaxlen = 100 # max number of words in a question to use\n\n## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d475a8f7a5959c9937eec6ad3bb6a0221eba0b4e"},"cell_type":"code","source":"# EMBEDDING_FILE = '../input/embeddings/GoogleNews-vectors-negative300/GoogleNews-vectors-negative300.bin'\n# embeddings_index = KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True)\n\nword_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = (np.random.rand(nb_words, embed_size) - 0.5) / 5.0\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    if word in embeddings_index:\n        embedding_vector = embeddings_index.get_vector(word)\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec0808b1b6f6239f60ed6837cbd338d14895d844"},"cell_type":"code","source":"embed_size = 300\ndef get_model():\n    inp = Input(shape=(maxlen,))\n    x = Embedding(max_features, embed_size, weights=[embedding_matrix])(inp)\n    # x = CuDNNGRU(64, return_sequences=True)(x)\n    x = Bidirectional(CuDNNGRU(64, return_sequences=True))(x)\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n    conc = concatenate([avg_pool, max_pool])\n    conc = Dense(64, activation=\"relu\")(conc)\n    conc = Dropout(0.1)(conc)\n    outp = Dense(1, activation=\"sigmoid\")(conc)\n    \n    model = Model(inputs=inp, outputs=outp)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['accuracy'])\n\n    return model\n\nmodel = get_model()\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1adbb4453c4a771ccf2c44e77ca2b17d1e3e445f"},"cell_type":"code","source":"epochs=2\nfor e in range(epochs):\n    model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))\n    pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\n    \n    best_thresh = 0.5\n    best_score = 0.0\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        score = metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))\n        if score > best_score:\n            best_thresh = thresh\n            best_score = score\n            \n    print(\"Val F1 Score: {:.4f}\".format(best_score))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ad0238d69808285850ada30250c6fadbd0887186"},"cell_type":"markdown","source":"**With new dataset**\n\nBe careful don't use fake data for validation"},{"metadata":{"trusted":true,"_uuid":"1109c471277a3eb4661b859d9e256b2afb11e8fd"},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\ntest_df = pd.read_csv(\"../input/test.csv\")\nprint(\"Train shape : \",train_df.shape)\nprint(\"Test shape : \",test_df.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a90e30682e256c132ba4508e558eb00244df5010"},"cell_type":"code","source":"## split to train and val\ntrain_df, val_df = train_test_split(train_df, test_size=0.1, random_state=2018)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d2f4d581be7d3006669770f748d50538aeb1d438"},"cell_type":"code","source":"## Add fake data\ntrain_fake = train_df[train_df.target == 1].sample(n=10000, replace=True)\ntrain_fake[\"real_question_text\"] = train_fake[\"question_text\"]\ntrain_fake[\"question_text\"] = train_fake[\"question_text\"].apply(lambda x: replace_words(x, 0.3))\n\ntrain_fake.head() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"622e91b53b94f065139a14af756a6acd678eba59"},"cell_type":"code","source":"train_df = train_df.append(train_fake.drop(columns=['real_question_text'])).sample(frac = 1.0, replace=False)\nlen(train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d7975b22cabba37781761a998f990ce751becbbd"},"cell_type":"code","source":"## fill up the missing values\ntrain_X = train_df[\"question_text\"].fillna(\"_na_\").values\nval_X = val_df[\"question_text\"].fillna(\"_na_\").values\ntest_X = test_df[\"question_text\"].fillna(\"_na_\").values\n\n## Tokenize the sentences\ntokenizer = Tokenizer(num_words=max_features)\ntokenizer.fit_on_texts(list(train_X))\ntrain_X = tokenizer.texts_to_sequences(train_X)\nval_X = tokenizer.texts_to_sequences(val_X)\ntest_X = tokenizer.texts_to_sequences(test_X)\n\n## Pad the sentences \ntrain_X = pad_sequences(train_X, maxlen=maxlen)\nval_X = pad_sequences(val_X, maxlen=maxlen)\ntest_X = pad_sequences(test_X, maxlen=maxlen)\n\n## Get the target values\ntrain_y = train_df['target'].values\nval_y = val_df['target'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9b9971df63f3349958ba26f26b4aa2d76c4a2667"},"cell_type":"code","source":"word_index = tokenizer.word_index\nnb_words = min(max_features, len(word_index))\nembedding_matrix = (np.random.rand(nb_words, embed_size) - 0.5) / 5.0\nfor word, i in word_index.items():\n    if i >= max_features: continue\n    if word in embeddings_index:\n        embedding_vector = embeddings_index.get_vector(word)\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3676a80345aa348c47174ba5fc7fe2220903a7e4"},"cell_type":"code","source":"model = get_model()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a362dfcd99d05b22213cd98493936dfd19adfd22"},"cell_type":"code","source":"epochs=2\nfor e in range(epochs):\n    model.fit(train_X, train_y, batch_size=512, epochs=1, validation_data=(val_X, val_y))\n    pred_glove_val_y = model.predict([val_X], batch_size=1024, verbose=1)\n    \n    best_thresh = 0.5\n    best_score = 0.0\n    for thresh in np.arange(0.1, 0.501, 0.01):\n        thresh = np.round(thresh, 2)\n        score = metrics.f1_score(val_y, (pred_glove_val_y>thresh).astype(int))\n        if score > best_score:\n            best_thresh = thresh\n            best_score = score\n            \n    print(\"Val F1 Score: {:.4f}\".format(best_score))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}