{"cells":[{"metadata":{"_uuid":"8f386ca4e7e9ef11523ef3993a7bce5432a2f91b"},"cell_type":"markdown","source":"# Some Remarks:\n1. The encoder only works for English sentences, so you'll need an alternative for other languages in the text\n2. This is merely a work in progress, but this is something to refer to when you're trying to install an external package without using the internet\n3. I will probably do some text cleaning in further versions"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!export NLTK_DATA=../input/infersent/punkt/punkt\n!ls /kaggle/input/infersentrepo/repository/facebookresearch-InferSent-940c003\n!cd /kaggle/working\nimport sys\nsys.path.append('/kaggle/input/infersentrepo/repository/facebookresearch-InferSent-940c003')\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nprint(os.listdir(\"/kaggle/input/infersentrepo/repository/facebookresearch-InferSent-940c003\"))\nimport nltk\n%load_ext autoreload\n%autoreload 2\n%matplotlib inline\nfrom random import randint\nimport numpy as np\nimport torch\nfrom models import InferSent\n!ls /kaggle/input","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"model_version = 1\nMODEL_PATH = \"/kaggle/input/infersent/infersent%s.pkl\" % model_version\nparams_model = {'bsize': 64, 'word_emb_dim': 300, 'enc_lstm_dim': 2048,\n                'pool_type': 'max', 'dpout_model': 0.0, 'version': model_version}\nmodel = InferSent(params_model)\nmodel.load_state_dict(torch.load(MODEL_PATH))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"498aeda2e5043a10c99cea7f0b962dd904de607d"},"cell_type":"code","source":"use_cuda = False\nmodel = model.cuda() if use_cuda else model\n# If infersent1 -> use GloVe embeddings. If infersent2 -> use InferSent embeddings.\nW2V_PATH = '/kaggle/input/quora-insincere-questions-classification/embeddings/glove.840B.300d/glove.840B.300d.txt' if model_version == 1 else ''\nmodel.set_w2v_path(W2V_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0cc423009063b090db59a048baf13d2226bf0b2b"},"cell_type":"code","source":"# Load embeddings of K most frequent words\nmodel.build_vocab_k_words(K=100000)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0c6b00ee77806c9029b582a8ca1002525ae99873"},"cell_type":"code","source":"a=pd.read_csv('/kaggle/input/petfinder-adoption-prediction/train/train.csv')\nb=a['Description'].values\nf = open(\"/kaggle/working/pet.txt\", \"w\")\nf = open(\"/kaggle/working/pet.txt\", \"a+\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"151b8a25a922fa333f4060a5787529b468ace7fb"},"cell_type":"code","source":"for i in range(10000):\n    f.write(str(b[i])+'\\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fc192235e4479ad85bb8eb33403b1fc5e185757f"},"cell_type":"code","source":"# Load some sentences\nsentences = []\nwith open('/kaggle/working/pet.txt') as f:\n    for line in f:\n        sentences.append(line.strip())\nprint(len(sentences))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ec6e4e59f639561f425801081f0d26ca78cf28fa"},"cell_type":"code","source":"sentences[:5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1b88a0324dcbbc34f2338e3d7a539660f252695d"},"cell_type":"code","source":"embeddings = model.encode(sentences, bsize=128, tokenize=False, verbose=True)\nprint('nb sentences encoded : {0}'.format(len(embeddings)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"93cc73ac938166b3460cbeb64ea1e37505832075"},"cell_type":"code","source":"len(model.encode(['the cat eats.'])[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"17bb9c7b8243c80cee65c5abc67aa66eb852de64"},"cell_type":"markdown","source":""}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}