{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import json\nimport os\nimport random\nfrom collections import Counter, defaultdict\nfrom functools import partial\nfrom multiprocessing import Pool\n\nimport numpy as np\nimport pandas as pd\nimport sklearn\nimport torch\nfrom gensim.models import Word2Vec, KeyedVectors\n\n%load_ext Cython","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5e7ac662c326bef0275562d32d331c57ac24930e"},"cell_type":"markdown","source":"# Library codes"},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":true,"_uuid":"d4d5c443056acc104300ca80bba1c3fcdfd81a78"},"cell_type":"code","source":"%%cython\nimport re\nfrom multiprocessing import Pool\n\nimport numpy as np\ncimport numpy as np\n\n\ncdef class StringReplacer:\n    cpdef public dict rule\n    cpdef list keys\n    cpdef list values\n    cpdef int n_rules\n\n    def __init__(self, dict rule):\n        self.rule = rule\n        self.keys = list(rule.keys())\n        self.values = list(rule.values())\n        self.n_rules = len(rule)\n\n    def __call__(self, str x):\n        cdef int i\n        for i in range(self.n_rules):\n            if self.keys[i] in x:\n                x = x.replace(self.keys[i], self.values[i])\n        return x\n\n    def __getstate__(self):\n        return (self.rule, self.keys, self.values, self.n_rules)\n\n    def __setstate__(self, state):\n        self.rule, self.keys, self.values, self.n_rules = state\n        \n        \ncdef class RegExpReplacer:\n    cdef dict rule\n    cdef list keys\n    cdef list values\n    cdef regexp\n    cdef int n_rules\n\n    def __init__(self, dict rule):\n        self.rule = rule\n        self.keys = list(rule.keys())\n        self.values = list(rule.values())\n        self.regexp = re.compile('(%s)' % '|'.join(self.keys))\n        self.n_rules = len(rule)\n\n    @property\n    def rule(self):\n        return self.rule\n\n    def __call__(self, str x):\n        def replace(match):\n            x = match.group(0)\n            if x in self.rule:\n                return self.rule[x]\n            else:\n                for i in range(self.n_rules):\n                    x = re.sub(self.keys[i], self.values[i], x)\n                return x\n        return self.regexp.sub(replace, x)\n    \n\ncdef class ApplyNdArray:\n    cdef func\n    cdef dtype\n    cdef dims\n    cdef int processes\n\n    def __init__(self, func, processes=1, dtype=object, dims=None):\n        self.func = func\n        self.processes = processes\n        self.dtype = dtype\n        self.dims = dims\n\n    def __call__(self, arr):\n        if self.processes == 1:\n            return self.apply(arr)\n        else:\n            return self.apply_parallel(arr)\n\n    cpdef apply(self, arr):\n        cdef int i\n        cdef int n = len(arr)\n        if self.dims is not None:\n            shape = (n, *self.dims)\n        else:\n            shape = n\n        cdef res = np.empty(shape, dtype=self.dtype)\n        for i in range(n):\n            res[i] = self.func(arr[i])\n        return res\n\n    cpdef apply_parallel(self, arr):\n        cdef list arrs = np.array_split(arr, self.processes)\n        with Pool(processes=self.processes) as pool:\n            outputs = pool.map(self.apply, arrs)\n        return np.concatenate(outputs, axis=0)\n","execution_count":null,"outputs":[]},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":false,"trusted":true,"_uuid":"f5f13a21c11253200978ad3f2a5a2612f2a18217"},"cell_type":"code","source":"def load_qiqc(n_rows=None):\n    train_df = pd.read_csv(f'{os.environ[\"DATADIR\"]}/train.csv', nrows=n_rows)\n    submit_df = pd.read_csv(f'{os.environ[\"DATADIR\"]}/test.csv', nrows=n_rows)\n    n_labels = {\n        0: (train_df.target == 0).sum(),\n        1: (train_df.target == 1).sum(),\n    }\n    train_df['target'] = train_df.target.astype('f')\n    train_df['weights'] = train_df.target.apply(lambda t: 1 / n_labels[t])\n\n    return train_df, submit_df\n\n\ndef build_datasets(train_df, submit_df, holdout, seed):\n    submit_dataset = QIQCDataset(submit_df)\n    if holdout:\n        # Train : Test split for holdout training\n        splitter = sklearn.model_selection.StratifiedShuffleSplit(\n            n_splits=1, test_size=0.1, random_state=seed)\n        train_indices, test_indices = list(splitter.split(\n            train_df, train_df.target))[0]\n        train_indices.sort(), test_indices.sort()\n        train_dataset = QIQCDataset(\n            train_df.iloc[train_indices].reset_index(drop=True))\n        test_dataset = QIQCDataset(\n            train_df.iloc[test_indices].reset_index(drop=True))\n    else:\n        train_dataset = QIQCDataset(train_df)\n        test_dataset = QIQCDataset(train_df.head(0))\n\n    return train_dataset, test_dataset, submit_dataset\n\n\nclass QIQCDataset(object):\n\n    def __init__(self, df):\n        self.df = df\n\n    @property\n    def tokens(self):\n        return self.df.tokens.values\n\n    @tokens.setter\n    def tokens(self, tokens):\n        self.df['tokens'] = tokens\n\n    @property\n    def positives(self):\n        return self.df[self.df.target == 1]\n\n    @property\n    def negatives(self):\n        return self.df[self.df.target == 0]\n\n    def build(self, device):\n        self._X = self.tids\n        self.X = torch.Tensor(self._X).type(torch.long).to(device)\n        if 'target' in self.df:\n            self._t = self.df.target[:, None]\n            self._W = self.df.weights\n            self.t = torch.Tensor(self._t).type(torch.float).to(device)\n            self.W = torch.Tensor(self._W).type(torch.float).to(device)\n        if hasattr(self, '_X2'):\n            self.X2 = torch.Tensor(self._X2).type(torch.float).to(device)\n        else:\n            self._X2 = np.zeros((self._X.shape[0], 1), 'f')\n            self.X2 = torch.Tensor(self._X2).type(torch.float).to(device)\n\n    def build_labeled_dataset(self, indices):\n        return torch.utils.data.TensorDataset(\n            self.X[indices], self.X2[indices],\n            self.t[indices], self.W[indices])\n    \n## Pretrained vector\n\ndef load_pretrained_vectors(names, token2id, test=False):\n    assert isinstance(names, list)\n    with Pool(processes=len(names)) as pool:\n        f = partial(load_pretrained_vector, token2id=token2id, test=test)\n        vectors = pool.map(f, names)\n    return dict([(n, v) for n, v in zip(names, vectors)])\n\n\ndef load_pretrained_vector(name, token2id, test=False):\n    loader = dict(\n        gnews=GNewsPretrainedVector,\n        wnews=WNewsPretrainedVector,\n        paragram=ParagramPretrainedVector,\n        glove=GlovePretrainedVector,\n    )\n    return loader[name].load(token2id, test)\n\n\nclass BasePretrainedVector(object):\n\n    @classmethod\n    def load(cls, token2id, test=False, limit=None):\n        embed_shape = (len(token2id), 300)\n        freqs = np.zeros((len(token2id)), dtype='f')\n\n        if test:\n            np.random.seed(0)\n            vectors = np.random.normal(0, 1, embed_shape)\n            vectors[0] = 0\n            vectors[len(token2id) // 2:] = 0\n        else:\n            vectors = np.zeros(embed_shape, dtype='f')\n            path = f'{os.environ[\"DATADIR\"]}/{cls.path}'\n            for i, o in enumerate(\n                    open(path, encoding=\"utf8\", errors='ignore')):\n                token, *vector = o.split(' ')\n                token = str.lower(token)\n                if token not in token2id or len(o) <= 100:\n                    continue\n                if limit is not None and i > limit:\n                    break\n                freqs[token2id[token]] += 1\n                vectors[token2id[token]] += np.array(vector, 'f')\n\n        vectors[freqs != 0] /= freqs[freqs != 0][:, None]\n        vec = KeyedVectors(300)\n        vec.add(list(token2id.keys()), vectors, replace=True)\n\n        return vec\n\n\nclass GNewsPretrainedVector(object):\n\n    name = 'GoogleNews-vectors-negative300'\n    path = f'embeddings/{name}/{name}.bin'\n\n    @classmethod\n    def load(cls, tokens, limit=None):\n        raise NotImplementedError\n        path = f'{os.environ[\"DATADIR\"]}/{cls.path}'\n        return KeyedVectors.load_word2vec_format(\n            path, binary=True, limit=limit)\n\n\nclass WNewsPretrainedVector(BasePretrainedVector):\n\n    name = 'wiki-news-300d-1M'\n    path = f'embeddings/{name}/{name}.vec'\n\n\nclass ParagramPretrainedVector(BasePretrainedVector):\n\n    name = 'paragram_300_sl999'\n    path = f'embeddings/{name}/{name}.txt'\n\n\nclass GlovePretrainedVector(BasePretrainedVector):\n\n    name = 'glove.840B.300d'\n    path = f'embeddings/{name}/{name}.txt'\n\n    \nclass WordVocab(object):\n\n    def __init__(self, mincount=1):\n        self.counter = Counter()\n        self.n_documents = 0\n        self._counters = {}\n        self._n_documents = defaultdict(int)\n        self.mincount = mincount\n\n    def __len__(self):\n        return len(self.token2id)\n\n    def add_documents(self, documents, name):\n        self._counters[name] = Counter()\n        for document in documents:\n            bow = dict.fromkeys(document, 1)\n            self._counters[name].update(bow)\n            self.counter.update(bow)\n            self.n_documents += 1\n            self._n_documents[name] += 1\n\n    def build(self):\n        counter = dict(self.counter.most_common())\n        self.word_freq = {\n            **{'<PAD>': 0},\n            **counter,\n        }\n        self.token2id = {\n            **{'<PAD>': 0},\n            **{word: i + 1 for i, word in enumerate(counter)}\n        }\n        self.lfq = np.array(list(self.word_freq.values())) < self.mincount\n        self.hfq = ~self.lfq\n        \n        \nclass PunctSpacer(StringReplacer):\n\n    def __init__(self, edge_only=False):\n        puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£', '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', '█', '½', '…', '“', '★', '”', '–', '●', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', '¯', '♦', '¤', '▲', '¸', '¾', '⋅', '‘', '∞', '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]  # NOQA\n        if edge_only:\n            rule = {\n                **dict([(f' {p}', f' {p} ') for p in puncts]),\n                **dict([(f'{p} ', f' {p} ') for p in puncts]),\n            }\n        else:\n            rule = dict([(p, f' {p} ') for p in puncts])\n        super().__init__(rule)\n        \n        \nclass NumberReplacer(RegExpReplacer):\n\n    def __init__(self, with_underscore=False):\n        prefix, suffix = '', ''\n        if with_underscore:\n            prefix += ' __'\n            suffix = '__ '\n        rule = {\n            '[0-9]{5,}': f'{prefix}#####{suffix}',\n            '[0-9]{4}': f'{prefix}####{suffix}',\n            '[0-9]{3}': f'{prefix}###{suffix}',\n            '[0-9]{2}': f'{prefix}##{suffix}',\n        }\n        super().__init__(rule)\n\n\ndef set_seed(seed=0):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n\n\nclass Pipeline(object):\n\n    def __init__(self, *modules):\n        self.modules = modules\n\n    def __call__(self, x):\n        for module in self.modules:\n            x = module(x)\n        return x","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"526221606c1148cfd46458060c1347e9cc07dbc2"},"cell_type":"markdown","source":"# Setup & preprocessing"},{"metadata":{"trusted":true,"_uuid":"5d80c92060baccb51ca6d628d1b8018ba7450eac"},"cell_type":"code","source":"%%time\nos.environ['DATADIR'] = '/kaggle/input'\nset_seed(0)\ntrain_df, submit_df = load_qiqc()\ndatasets = build_datasets(train_df, submit_df, holdout=False, seed=0)\ntrain_dataset, test_dataset, submit_dataset = datasets","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"09ff98a561dc53018b100c0295b78f0d2fd3a22a"},"cell_type":"code","source":"%%time\ntokenize = Pipeline(\n    str.lower,\n    PunctSpacer(),\n    NumberReplacer(with_underscore=True),\n    str.split\n)\napply_tokenize = ApplyNdArray(tokenize, processes=2, dtype=object)\ntrain_dataset.tokens, test_dataset.tokens, submit_dataset.tokens = \\\n    [apply_tokenize(d.df.question_text.values) for d in datasets]\ntokens = np.concatenate([d.tokens for d in datasets])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7b9b2d843c3d19cb4b4cdf7a6b2d0ef3dee95dcc"},"cell_type":"code","source":"%%time\nvocab = WordVocab(mincount=1)\nvocab.add_documents(train_dataset.positives.tokens, 'train-pos')\nvocab.add_documents(train_dataset.negatives.tokens, 'train-neg')\nvocab.add_documents(test_dataset.positives.tokens, 'test-pos')\nvocab.add_documents(test_dataset.negatives.tokens, 'test-neg')\nvocab.add_documents(submit_dataset.df.tokens, 'submit')\nvocab.build()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"c8970a98d6d742a43eee469e8fa0544b9b7107af"},"cell_type":"code","source":"%%time\nglove = load_pretrained_vector('glove', vocab.token2id)\nword_vectors = {'glove': glove}\nunk = (glove.vectors == 0).all(axis=1)\nknown = ~unk","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"fde63b044c18ecc7da643d5599b80896ce5d14a5"},"cell_type":"code","source":"params = dict(\n    min_count=1,\n    workers=1,\n    iter=5,\n    size=300,\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5462ab473010cfdfb1be13d772588bddb69a7cb3"},"cell_type":"markdown","source":"# Build models & training"},{"metadata":{"_uuid":"4407b9bb90f2cd75d0adefdc60c22e35f3b2ba22"},"cell_type":"markdown","source":"## Word2Vec scratch"},{"metadata":{"trusted":true,"_uuid":"53099019341efb343ab677fecc083854798ec1be"},"cell_type":"code","source":"%%time\nmodel = Word2Vec(**params)\nmodel.build_vocab_from_freq(vocab.word_freq)\nmodel.train(tokens, total_examples=len(tokens), epochs=model.epochs)\nword_vectors['scratch'] = model.wv","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"559b4af06aaf4dee5b58592aa3c90dda51a60fc0"},"cell_type":"markdown","source":"## Word2Vec fine-tuning (word vector & context vector)"},{"metadata":{"trusted":true,"_uuid":"a10b465a96b0918cadb9529c5f82e17d30eec99b"},"cell_type":"code","source":"%%time\nmodel = Word2Vec(**params)\nmodel.build_vocab_from_freq(vocab.word_freq)\nidxmap = np.array(\n    [vocab.token2id[w] for w in model.wv.index2entity])\nmodel.wv.vectors[:] = glove.vectors[idxmap]\nmodel.trainables.syn1neg[:] = glove.vectors[idxmap]\nmodel.train(tokens, total_examples=len(tokens), epochs=model.epochs)\nword_vectors['finetune'] = model.wv","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"873402089102b141278d1f619f4ff5376dbb63af"},"cell_type":"markdown","source":"# Evaluations"},{"metadata":{"_uuid":"474bff328454c1015f63167540e130236614e5c2"},"cell_type":"markdown","source":"## **High** frequency words in Quora & **known** words in Glove\n\n- Glove: ○\n- Scratch: ○\n- Finetune: ○"},{"metadata":{"trusted":true,"_uuid":"0c441917ddae54758ba16415ed752e3cac057397"},"cell_type":"code","source":"word = 'obama'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cae423e3bc287b349626dde267eb357b7aee8c62"},"cell_type":"code","source":"word = 'lgbt'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e368521661c75e16a66efca6ac4944ee2b21995"},"cell_type":"code","source":"word = 'cosx'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9b04f0d3bfe8239f73e7fa264704481dc0ea9242"},"cell_type":"markdown","source":"## **High** frequency words in Quora & **unknown** words in Glove\n\n- Glove: -\n- Scratch: ○\n- Finetune: ○"},{"metadata":{"trusted":true,"_uuid":"7b4d29ebb5afc6f0a0855d25214a2cf660e49600"},"cell_type":"code","source":"word = 'brexit'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5b1176a65df4b0c1d21c12ce64542609f71ccbb5"},"cell_type":"code","source":"word = 'coinbase'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"36030063950be965074b1a5315cf6c1f042773a4"},"cell_type":"code","source":"word = 'tensorflow'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"73c705747793d25cc47e48b650e436fa0b6faf4a"},"cell_type":"code","source":"word = 'cos2x'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"bc6a79c73a641e811c2fc2e075df6a08fa67275f"},"cell_type":"code","source":"word = 'kubernetes'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"809d1f0acc3158271a24c453f3070514c6512fb8"},"cell_type":"code","source":"word = 'gdpr'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ff3c50006c20323a26b78d8016c0d91365901381"},"cell_type":"markdown","source":"## **Low** frequency words in Quora & **known** words in Glove\n\n- Glove: ○\n- Scratch: ☓\n- Finetune: ○"},{"metadata":{"trusted":true,"_uuid":"cf21c2ea0c597e6550f5331c548df31744654010"},"cell_type":"code","source":"word = '0bama'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ca932288ea0f76e806e39db173fd6a1415229b4"},"cell_type":"code","source":"word = 'germnay'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4a0b62e62b89768608386ec8f4d36244c7577199"},"cell_type":"code","source":"word = 'gogole'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9a19d5e5f868e22964d969c2344a36c673ab3060"},"cell_type":"code","source":"word = 'javadoc'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"d82e25312cd88c7c8423d3b96ba2eb10b775a99c"},"cell_type":"code","source":"word = 'cython'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9ed7cbde0ee9b36b44756beacd5f8435deab2db4"},"cell_type":"code","source":"word = 'compresses'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e8c41623e906d60553fad691df22e7393367c300"},"cell_type":"markdown","source":"## **Low** frequency words in Quora & **unknown** words in Glove\n- Glove: ☓\n- Scratch: ☓\n- Finetune: ☓"},{"metadata":{"trusted":true,"_uuid":"197bd83bf779467eeb451f15f6416371ea82e05a"},"cell_type":"code","source":"word = 'xgboost'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b8d6765826915230422793208a95bfe6cdcdd093"},"cell_type":"code","source":"word = '2sinxcosx'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9e6e26dd7b9e9888231a26e0b7a8a63952b9e0b9"},"cell_type":"code","source":"word = 'germeny'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2515f6ef7fc7bd29b3843b30912747e38bd55221"},"cell_type":"code","source":"word = 'bigender'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"486847a11494023dd3e0574cf4d0760b3c61b1e2"},"cell_type":"code","source":"word = 'youcanttellyourstoryfromthe'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"1e6c1529241e262c8e7aa74025111e2d7f9d1999"},"cell_type":"code","source":"word = '5gfwdhf4rz'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"f9acb90b3b901feb233e5a9dce8ba5e0653f3733"},"cell_type":"code","source":"word = 'ॡ'\nprint(vocab.word_freq[word])\npd.DataFrame({name: kv.most_similar(word) for name, kv in word_vectors.items()})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ae0377d01c992fddc1c4db14d387361dcf6136c6"},"cell_type":"code","source":"pd.DataFrame(np.array(list(vocab.word_freq.items()))).to_csv('all.csv', index=False, sep='\\t')\npd.DataFrame(np.array(list(vocab.word_freq.items()))[unk]).to_csv('unk.csv', index=False, sep='\\t')\npd.DataFrame(np.array(list(vocab.word_freq.items()))[known]).to_csv('known.csv', index=False, sep='\\t')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"4b6911e9f54df2e61c227505bc2c29602c54ee3d"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}