{"cells":[{"metadata":{"_uuid":"f39377548589326f677d2bc4a5694198ebc7ba0e"},"cell_type":"markdown","source":"Simple kernel for how to quickly calculate similarity of words from embeddings using GPU and tensorflow."},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\nimport time\nimport numpy as np \nimport pandas as pd \n\nimport math\nfrom sklearn.model_selection import train_test_split\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(\"../input/train.csv\")\n\nprint(\"Train shape : \",train_df.shape)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"74224cef51d3bccaf5e89169ada94cc5c3af5275"},"cell_type":"code","source":"X_train = train_df[\"question_text\"].fillna(\"_na_\").values\n\ntokenizer = Tokenizer(num_words=100_000)\ntokenizer.fit_on_texts(list(X_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2b34ab16d6f0dee4fed6ca442121e0439289d4c4"},"cell_type":"code","source":"EMBEDDING_FILE = '../input/embeddings/glove.840B.300d/glove.840B.300d.txt'\ndef get_coefs(word,*arr): return word, np.asarray(arr, dtype='float32')\nembeddings_index = dict(get_coefs(*o.split(\" \")) for o in open(EMBEDDING_FILE))\n\nall_embs = np.stack(embeddings_index.values())\nemb_mean,emb_std = all_embs.mean(), all_embs.std()\nembed_size = all_embs.shape[1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"84f87872f2f66da1444bb317d8989f32de7d4bda"},"cell_type":"code","source":"nb_words = 100_000\nembedding_matrix_glove = np.random.normal(emb_mean, emb_std, (nb_words, embed_size))\n\nfor word, i in tokenizer.word_index.items():\n    if i >= 100_000:\n        continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None: \n        embedding_matrix_glove[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"59c88a2c030bea1b5f05641e55436aef00ef7e7b"},"cell_type":"code","source":"# adapted from https://stackoverflow.com/questions/37558899/efficiently-finding-closest-word-in-tensorflow-embedding\nimport tensorflow as tf\n\nbatch_size = 10_000\nn_neighbors = 10\nclosest_words = np.zeros((nb_words, n_neighbors+1))\n\nembedding = tf.placeholder(tf.float32, [nb_words, embed_size])\nbatch_array = tf.placeholder(tf.float32, [batch_size, embed_size])\nnormed_embedding = tf.nn.l2_normalize(embedding, dim=1)\nnormed_array = tf.nn.l2_normalize(batch_array, dim=1)\ncosine_similarity = tf.matmul(normed_array, tf.transpose(normed_embedding))\nclosest_k_words = tf.nn.top_k(cosine_similarity,k=n_neighbors+1)\n\nwith tf.Session() as session:\n    start_idx = 0\n    for end_idx in range(batch_size, nb_words, batch_size):\n        print(end_idx)\n        result = session.run(closest_k_words, feed_dict={embedding: embedding_matrix_glove, batch_array: embedding_matrix_glove[start_idx:end_idx]})\n        closest_words[start_idx:end_idx] = result[1]\n\n        start_idx = end_idx","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"30f36a10b8022732b43430e65a6302c56351560a"},"cell_type":"code","source":"index_to_word = {v:k for k,v in tokenizer.word_index.items()}\nindex_to_word[0] = \"<PAD>\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22f1acb72575c85b1a58ea33618b9fd9ecb4b825"},"cell_type":"code","source":"synonyms = {index_to_word[int(x[0])]: [index_to_word[int(y)] for y in x[1:]] for x in closest_words}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cab6282e1e4d262aa8f82140cf7c9901deee2c2"},"cell_type":"code","source":"synonyms[\"king\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0221a5f02d69628d04b5985c012b2374b2a7063e"},"cell_type":"code","source":"synonyms[\"quora\"]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5ada0fe5dc473bce9d7075c16edda8ab126f541a"},"cell_type":"markdown","source":"I have tried a few things for data augmentation, without any luck. Maybe someone has some ideas how to use it."},{"metadata":{"trusted":true,"_uuid":"690f344ef5f37f3ac810ea82f4f70dc86d3cd9e0"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}