{ "cells": [ { "cell_type": "markdown", "metadata": { "_cell_guid": "ffaf2a4b-6a0c-2ba2-769f-450e2914d958" }, "source": [ "A beginners word2vec implementation demonstrated here. \n", "\n", "P.S Its so naive that I use the csv library to read and write data to csv files. Switching to pandas as we speak." ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "f9088559-6a3f-59c5-daed-abfb8f168250" }, "outputs": [], "source": [ "import math, csv\n", "import heapq\n", "from nltk.tokenize import word_tokenize\n", "from nltk.corpus.reader.wordnet import ADJ, ADJ_SAT, ADV, NOUN, VERB\n", "from nltk.stem import WordNetLemmatizer\n", "import time # For computing running time\n", "import gensim\n", "import numpy as np\n", "\n", "from subprocess import check_output\n", "print(check_output([\"ls\", \"../input\"]).decode(\"utf8\"))\n", "\n", "# Any results you write to the current directory are saved as output." ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "8d52eea8-863a-9e6e-90e1-d4a231fd1237" }, "source": [ "Using this custom stopwords frozen set because I had this made from an NLP project in a similar domain !" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "10867a6d-da3c-85db-2b1b-2f4806be5ba7" }, "outputs": [], "source": [ "stopwords = frozenset(\n", " \t[u'i', u'me', u'my', u'myself', u'we', u'our', u'ours', u'ourselves', u'you', u'your', u'yours',\n", " u'yourself', u'yourselves', u'he', u'him', u'his', u'himself', u'she', u'her', u'hers', u'herself', u'it',\n", " u'its', u'itself', u'they', u'them', u'their', u'theirs', u'themselves', u'what', u'which', u'who',\n", " u'whom', u'this', u'that', u'these', u'those', u'am', u'is', u'are', u'was', u'were', u'be', u'been',\n", " u'being', u'have', u'has', u'had', u'having', u'do', u'does', u'did', u'doing', u'a', u'an', u'the',\n", " u'and', u'but', u'if', u'or', u'because', u'as', u'until', u'while', u'of', u'at', u'by', u'for', u'with',\n", " u'about', u'against', u'between', u'into', u'through', u'during', u'before', u'after', u'above', u'below',\n", " u'to', u'from', u'up', u'down', u'in', u'out', u'on', u'off', u'over', u'under', u'again', u'further',\n", " u'then', u'once', u'here', u'there', u'when', u'where', u'why', u'how', u'all', u'any', u'both', u'each',\n", " u'few', u'more', u'most', u'other', u'some', u'such', u'no', u'nor', u'not', u'only', u'own', u'same',\n", " u'so', u'than', u'too', u'very', u's', u't', u'can', u'will', u'just', u'don', u'should', u'now', u\"'s\", u'?', u'50/50'])\n" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "7027ba32-475a-4107-b900-acaebfd384f8" }, "source": [ ":function cleaner :: Returns the word2vec vector of the question. Simple median of the vectors of all the non-stop words in the question.\n", "\n", ":function numpy_cosine:: Returns cosine similarity of two vectors" ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "df3cc83c-4a25-c73f-7b08-5ccbb82185e1" }, "outputs": [], "source": [ "def cleaner(question_string):\n", "\t\"\"\"\n", "\tGiven question string, returns word2vec vector of the questions tring\n", "\t:param question_string : The given question as a string.\n", "\t\"\"\"\n", "\twords = word_tokenize(question_string)[:-1]\n", "\tnon_stop_words = []\n", "\tfor w in words:\n", "\t\tif w.lower().strip('-') not in stopwords and w.lower() in model.vocab:\n", "\t\t\tw = WordNetLemmatizer().lemmatize(w, NOUN)\n", "\t\t\tnon_stop_words.append(w.lower().strip('-'))\n", "\t#print non_stop_words\n", "\tvectors = [model[word] for word in non_stop_words]\n", "\tvector = sum(vectors)/float(len(non_stop_words))\n", "\treturn vector\n", "\n", "def numpy_cosine(q1_vec, q2_vec):\n", "\t\"\"\"\n", "\tCosine similarity between q1 and q2 question instances using their vectors\n", "\t:param q1_vec: cleaner(question1)\n", "\t:param q2_vec: cleaner(question2)\n", "\t:return: similarity between q1 and q2\n", "\t\"\"\"\n", "\t#\tprint q1_vec\n", "\tcosine_similarity = np.dot(q1_vec, q2_vec)/(np.linalg.norm(q1_vec)*np.linalg.norm(q2_vec))\n", "\t#\tprint np.dot(q1_vec, q2_vec)\n", "\t#\tprint type(np.dot(q1_vec, q2_vec))\n", "\treturn cosine_similarity\n" ] }, { "cell_type": "markdown", "metadata": { "_cell_guid": "e8388651-a1e6-14fc-6bfd-d5bc5f01b557" }, "source": [ "Please make sure to point it to the pre-loaded model in your local computer. Also I used the csv library to read and write the data which is deprecated now. Please use pandas for the same. " ] }, { "cell_type": "code", "execution_count": null, "metadata": { "_cell_guid": "ce94cd54-c7c2-86c5-28d9-7ef666522005" }, "outputs": [], "source": [ "MODEL_Googlenews_DIR = 'GoogleNews-vectors-negative300.bin'\n", "model = gensim.models.Word2Vec.load_word2vec_format(MODEL_Googlenews_DIR, binary=True)\n", "opener = open('test.csv', 'r')\n", "reader = csv.reader(opener)\n", "\n", "with open('output.csv', 'wb') as resultFile:\n", "\twr = csv.writer(resultFile, dialect='excel')\n", "\twr.writerow(['test_id', 'is_duplicate'])\n", "\theader = opener.readline()\n", "\tfor line in reader:\n", "\t\tqid = line[0]\n", "\t\tquestion1 = line[1]\n", "\t\tquestion2 = line[2]\n", "\t\ttry:\n", "\t\t\twr.writerow([int(qid), numpy_cosine(cleaner(question1), cleaner(question2))])\n", "\t\texcept:\n", "\t\t\tif qid=='test_id':\n", "\t\t\t\tpass\n", "\t\t\twr.writerow([ int(qid), 1 ])" ] } ], "metadata": { "_change_revision": 0, "_is_fork": false, "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.0" } }, "nbformat": 4, "nbformat_minor": 0 }