{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import cudf\nimport cuml\nimport pandas as pd\nimport numpy as np\nfrom cuml.manifold import TSNE\nimport matplotlib.pyplot as plt\nfrom gensim.models import Word2Vec,KeyedVectors\nfrom tqdm.auto import tqdm","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2023-01-21T02:36:14.060475Z","iopub.execute_input":"2023-01-21T02:36:14.061442Z","iopub.status.idle":"2023-01-21T02:36:18.412747Z","shell.execute_reply.started":"2023-01-21T02:36:14.061335Z","shell.execute_reply":"2023-01-21T02:36:18.411793Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%env PYTHONHASHSEED=1\nimport hashlib\ndef hashf(s):\n    return int(hashlib.md5(str(s).encode()).hexdigest(), 32)","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:36:18.414764Z","iopub.execute_input":"2023-01-21T02:36:18.415477Z","iopub.status.idle":"2023-01-21T02:36:18.423821Z","shell.execute_reply.started":"2023-01-21T02:36:18.415438Z","shell.execute_reply":"2023-01-21T02:36:18.422832Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model = Word2Vec.load(\"/kaggle/input/live-visualization-of-word2vec-loss-5-33-5-34/word2vec.model\")\naid2idx = {aid: i for i, aid in enumerate(model.wv.index_to_key)}\nembs = []\naids = []\nemb_size = len(model.wv.get_vector(aid2idx[0]))\nfor aid in tqdm(aid2idx.keys()):\n    #print(aid)\n    aids.append(aid)\n    embs.append(model.wv.get_vector(aid2idx[aid]))\ndf = pd.DataFrame(embs, columns=[f'w2emb_{i}' for i in range(emb_size)])\ndf  = cudf.from_pandas(df)","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:36:18.425110Z","iopub.execute_input":"2023-01-21T02:36:18.425993Z","iopub.status.idle":"2023-01-21T02:37:42.334709Z","shell.execute_reply.started":"2023-01-21T02:36:18.425957Z","shell.execute_reply":"2023-01-21T02:37:42.333652Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time \n\ntsne    = TSNE(n_components=2, perplexity=50, learning_rate=20)\ntsne_2D = tsne.fit_transform(df)","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:37:42.337610Z","iopub.execute_input":"2023-01-21T02:37:42.337997Z","iopub.status.idle":"2023-01-21T02:40:44.568123Z","shell.execute_reply.started":"2023-01-21T02:37:42.337957Z","shell.execute_reply":"2023-01-21T02:40:44.567008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tsne_2D","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:40:44.569763Z","iopub.execute_input":"2023-01-21T02:40:44.570138Z","iopub.status.idle":"2023-01-21T02:40:44.662700Z","shell.execute_reply.started":"2023-01-21T02:40:44.570101Z","shell.execute_reply":"2023-01-21T02:40:44.661545Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"len(aids)","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:40:44.664049Z","iopub.execute_input":"2023-01-21T02:40:44.664512Z","iopub.status.idle":"2023-01-21T02:40:44.671301Z","shell.execute_reply.started":"2023-01-21T02:40:44.664459Z","shell.execute_reply":"2023-01-21T02:40:44.670246Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tsne_2D.as_matrix().T","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:33:10.665743Z","iopub.execute_input":"2023-01-21T02:33:10.666069Z","iopub.status.idle":"2023-01-21T02:33:10.679055Z","shell.execute_reply.started":"2023-01-21T02:33:10.666036Z","shell.execute_reply":"2023-01-21T02:33:10.677946Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, y = tsne_2D.as_matrix().T","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:33:10.680467Z","iopub.execute_input":"2023-01-21T02:33:10.680809Z","iopub.status.idle":"2023-01-21T02:33:10.690820Z","shell.execute_reply.started":"2023-01-21T02:33:10.680775Z","shell.execute_reply":"2023-01-21T02:33:10.689820Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x, y = tsne_2D.as_matrix().T\nfig, ax = plt.subplots(figsize=(18, 18))\nax.scatter(x, y, s=0.1, c=x, cmap=plt.cm.plasma)\nax.set_title('t-SNE plot for all 130 features', fontsize=18)\nplt.show();","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:33:10.692471Z","iopub.execute_input":"2023-01-21T02:33:10.692883Z","iopub.status.idle":"2023-01-21T02:33:28.718246Z","shell.execute_reply.started":"2023-01-21T02:33:10.692845Z","shell.execute_reply":"2023-01-21T02:33:28.717431Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tsne_2D.columns = ['x','y']\ntsne_2D['aid'] = aids\ntsne_2D.to_parquet('w2vec_2d.parquet')","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:40:44.672859Z","iopub.execute_input":"2023-01-21T02:40:44.673487Z","iopub.status.idle":"2023-01-21T02:40:45.013597Z","shell.execute_reply.started":"2023-01-21T02:40:44.673452Z","shell.execute_reply":"2023-01-21T02:40:45.012570Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"\ntsne_2D","metadata":{"execution":{"iopub.status.busy":"2023-01-21T02:40:45.015050Z","iopub.execute_input":"2023-01-21T02:40:45.015674Z","iopub.status.idle":"2023-01-21T02:40:45.090420Z","shell.execute_reply.started":"2023-01-21T02:40:45.015627Z","shell.execute_reply":"2023-01-21T02:40:45.089454Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"","metadata":{},"execution_count":null,"outputs":[]}]}