{"cells":[{"metadata":{"_cell_guid":"799ee576-1a6f-41d7-8d87-b9d37331a9fd","_uuid":"54f3c59d884f259219a669087e76b615a2bca7e0"},"cell_type":"markdown","source":" Let's explore Russki!!
\n
"},{"metadata":{"_cell_guid":"ed497eef-295b-4670-adff-84f7beacee6f","_uuid":"7df6589241aaeedccb3f3a300b92cf16036f5130"},"cell_type":"markdown","source":"## This Notebook will show:\n1. How to read in word vectors\n2. How to create vector representation for each row of the \"Description\" Column\n\nPS. You can access the file here https://s3.us-east-2.amazonaws.com/datafaculty/train_desc_features.npy"},{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","collapsed":true,"trusted":true},"cell_type":"code","source":"import os\nimport pandas as pd\nimport numpy as np\nimport glob\nimport nltk\nimport gensim","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"387e00cb-6106-4ba3-b489-0d6308eec908","_uuid":"ef1b5123ba2289371296ddf0e1c21e08203b624a"},"cell_type":"markdown","source":"### Data import and usual stuff!!!"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","collapsed":true,"trusted":true},"cell_type":"code","source":"train=pd.read_csv(\"../input/avito-demand-prediction/train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"345f4850-55dc-4811-94d7-5a665d32aafd","_uuid":"05b1c13f80a0705fc9a11f4fae9bc402a642a2d7","collapsed":true,"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"21e2fcd4-43f9-47f4-98d9-40ad09cc4003","_uuid":"bffe002278eaebaacd398a01a76dd52b56ce8bb8"},"cell_type":"markdown","source":"### Let's load the word vectors"},{"metadata":{"_cell_guid":"af63cc4a-9d1b-45f2-a327-3d94ae6ceb8b","_uuid":"8e37cc47eaffeb94d482f05f019f8fed3a94e15c","collapsed":true,"trusted":true},"cell_type":"code","source":"from gensim.models import KeyedVectors","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"ed861792-f162-4aea-b4ed-605481a1580b","_uuid":"7b5d332ab43f6fd86451d35c6fba4c344315fba1","collapsed":true,"trusted":true},"cell_type":"code","source":"ru_model = KeyedVectors.load_word2vec_format('../input/fasttext-russian-2m/wiki.ru.vec')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4f9e8eca-cdc3-42d8-9b40-631efe290a75","_uuid":"f77a0f40b2cb0afd10b060862717b510d30fe59e","collapsed":true,"trusted":true},"cell_type":"code","source":"print(\"The size of vocabulary for this corpus is {}\".format(len(ru_model.vocab)))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"498860db-cb76-4b9e-bc73-4ca5b6346e59","_uuid":"149992e419fd2fcc964c023dbd3712a7e60b15d6"},"cell_type":"markdown","source":"### Let's explore these vectors and have some fun"},{"metadata":{"_cell_guid":"d30d97f2-593a-4da1-be7e-2f59b40dacff","_uuid":"717edd36510c39d029646175da6febec524e7129","collapsed":true,"trusted":true},"cell_type":"code","source":"# Pick a word \nfind_similar_to = 'Автомобили'.lower()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8ea8d9d7-5a9b-41e2-925c-acb12b86a7a5","_uuid":"81d50ba59d5d144b78f97298971faf3af0079ad7","collapsed":true,"trusted":true},"cell_type":"code","source":"ru_model.similar_by_word(find_similar_to)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"37e47e49-7b0c-41d9-acea-f9d6e2bb458b","_uuid":"796dc331f7b1d050ee9e4d6ddff86b16d6d4c980"},"cell_type":"markdown","source":"### Using yandex translate let's analyze the results:\n* We searched for Автомобили, which is Cars in English\n* The cosine matches were===========================>\n* aвтомобили---->Cars\n* микроавтомобили ----->Midget Car\n* автомобили\\xa0----->Cars\n* автомобили»----->Cars»\n* легковые----->Automobile\n* автомобили------>Cars\n* мотоциклы----->Motorcycles\n* спецавтомобили----->Special Vehicles\n* грузовики----->Trucks\n* автомобилевозы-----> Car Carrier\n\n"},{"metadata":{"_cell_guid":"3596b4e8-6cc5-4242-8e61-dac6c695cb2d","_uuid":"09218054e8cf275c43737b0d06b8d16777cfed30"},"cell_type":"markdown","source":"### Let's get back to business and create features from 'Description' column by adding word vectors"},{"metadata":{"_cell_guid":"e36e79c0-56ef-47eb-9ad7-b0c62aebb33c","_uuid":"a8c9f671ad3b375d23fb01a0e578c37b54a78f03","collapsed":true,"trusted":true},"cell_type":"code","source":"import nltk\ndef tokenize(x):\n '''Input: One description'''\n tok=nltk.tokenize.toktok.ToktokTokenizer()\n return [t.lower() for t in tok.tokenize(x)]\ndef get_vector(x):\n '''Input: Single token''' #If the word is out of vocab, then return a 300 dim vector filled with zeros\n try:\n return ru_model.get_vector(x)\n except:\n return np.zeros(shape=300)\ndef vector_sum(x):\n '''Input:List of word vectors'''\n return np.sum(x,axis=0)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"c2ef2916-6844-407e-9f8d-efd73277549c","_uuid":"07b746567fcf7da94907e08ed0db1ee12d025a52","collapsed":true,"trusted":true},"cell_type":"code","source":"features=[]\nfor desc in train['description'].values:\n tokens=tokenize(desc)\n if len(tokens)!=0: ## If the description is missing then return a 300 dim vector filled with zeros\n word_vecs=[get_vector(w) for w in tokens]\n features.append(vector_sum(word_vecs))\n else:\n features.append(np.zeros(shape=300)) ","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"e3f6bf27-67b8-45f1-9c89-df3c4e9a76f5","_uuid":"ffa33ba65973444ef358f0c576e94632cc1355fb","collapsed":true,"trusted":true},"cell_type":"code","source":"print(\"Features were extracted from {} rows\".format(len(features)))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"85f5e698-48cc-4e6c-b40c-2e0380bb3db0","_uuid":"e0cb681f996e080a34be153ba5dbbc04441f32e9","collapsed":true,"trusted":true},"cell_type":"code","source":"## Convert into numpy array\ntrain_desc_features=np.array(features)\nprint(\"Shape of features extracted from 'Description' column is:\")\nprint(train_desc_features.shape)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"61b10510c2c2fc9485d7a0614303e2ba484d59df"},"cell_type":"markdown","source":"## As can now be seen, we now have a dense representation of text. This representation can be used to build an Xgboost or Catboost model, taking into account both text data and regular columns. Such a model may perform better than one with tfidf features only."},{"metadata":{"_cell_guid":"e8bd4ed9-8ee5-4520-ae83-7173d90ed220","_uuid":"29147ecb01225330efdcf848ad0f6fd2852af4a5","collapsed":true,"trusted":true},"cell_type":"code","source":"## Write out as .npy file to be used later for modelling\n## np.save(\"train_desc_features.npy\",train_desc_features)\n## Due to kernel limitations, this step fails, I had trained a file locally and can be accessed from:\n## https://s3.us-east-2.amazonaws.com/datafaculty/train_desc_features.npy","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"98805331-d84b-4af9-9599-b8e8a746a9de","_uuid":"231af561de64fa043775321e4636891edc00a198","collapsed":true,"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.5","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}