{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":1,"outputs":[{"output_type":"stream","text":"['fasttext-english-word-vectors-including-subwords', 'imet-2019-fgvc6']\n","name":"stdout"}]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tqdm import tqdm\n\nfasttext_path = \"../input/fasttext-english-word-vectors-including-subwords/wiki-news-300d-1M-subword.vec\"\n\ndef load_vecs(word, *arr):\n    return (word, np.asarray(arr, dtype='float32'))\n\nvec_dic = dict(load_vecs(*line.rstrip().rsplit(' ')) for line in tqdm(open(fasttext_path)))","execution_count":null,"outputs":[{"output_type":"stream","text":"124445it [00:12, 9859.45it/s]","name":"stderr"}]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"labels = pd.read_csv(\"../input/imet-2019-fgvc6/labels.csv\")\nlabels.sample(5, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels[\"type\"] = labels[\"attribute_name\"].map(lambda x: x.split(\"::\")[0])\nlabels[\"name\"] = labels[\"attribute_name\"].map(lambda x: x.split(\"::\")[1])\nlabels.sample(5, random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tag_names = labels[labels[\"type\"] == \"tag\"][\"name\"].values\nculture_names = labels[labels[\"type\"] == \"culture\"][\"name\"].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_vec(w):\n    try:\n        return vec_dic[w]\n    except KeyError:\n        return np.zeros(300)\n    \ntag_vecs = []\nfor n in tag_names:\n    vecs = [get_vec(w) for w in n.split()]\n    vec = sum(vecs)/len(vecs)\n    tag_vecs.append(vec)\ntag_vecs = np.array(tag_vecs)\n\nculture_vecs = []\nfor n in culture_names:\n    vecs = [get_vec(w) for w in n.split()]\n    vec = sum(vecs)/len(vecs)\n    culture_vecs.append(vec)\nculture_vecs = np.array(culture_vecs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.manifold import TSNE\n\ntag_model = TSNE(n_components=2, random_state=42)\nnp.set_printoptions(suppress=True)\ntag_model.fit_transform(tag_vecs)\n\nculture_model = TSNE(n_components=2, random_state=42)\nnp.set_printoptions(suppress=True)\nculture_model.fit_transform(culture_vecs)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# tag"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(40,40))\nplt.scatter(tag_model.embedding_[:, 0], tag_model.embedding_[:,1])\n\ncount = 0\nfor label, x, y in zip(tag_names, tag_model.embedding_[:, 0], tag_model.embedding_[:, 1]):\n    count +=1\n    plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# culture"},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(figsize=(40,40))\nplt.scatter(culture_model.embedding_[:, 0], culture_model.embedding_[:,1])\n\ncount = 0\nfor label, x, y in zip(culture_names, culture_model.embedding_[:, 0], culture_model.embedding_[:, 1]):\n    count +=1\n    plt.annotate(label, xy=(x, y), xytext=(0, 0), textcoords='offset points')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}