{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"markdown","source":"## NoteBook Objective\nVisualization to understand data structure\n\nCredit: Some of the code was inspired by \n- https://www.kaggle.com/jeffd23/visualizing-word-vectors-with-t-sne"},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import pandas as pd\npd.options.mode.chained_assignment = None \nimport numpy as np\nimport re\nimport nltk\n\nfrom gensim.models import word2vec\n\nfrom sklearn.manifold import TSNE\nimport matplotlib.pyplot as plt\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"db69a3515929b9f206eaae2b78730733f2de09ef"},"cell_type":"code","source":"data = pd.read_csv('../input/train.csv').sample(50000, random_state=23)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2618a14e2fdf108530cdee10e78cba7139361348"},"cell_type":"code","source":"data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bc7861c65a522024ce3038109606fbbb835aac1"},"cell_type":"code","source":"STOP_WORDS = nltk.corpus.stopwords.words()\n\ndef clean_sentence(val):\n    \"remove chars that are not letters or numbers, downcase, then remove stop words\"\n    regex = re.compile('([^\\s\\w]|_)+')\n    sentence = regex.sub('', val).lower()\n    sentence = sentence.split(\" \")\n    \n    for word in list(sentence):\n        if word in STOP_WORDS:\n            sentence.remove(word)  \n            \n    sentence = \" \".join(sentence)\n    return sentence\n\ndef clean_dataframe(data):\n    \"drop nans, then apply 'clean_sentence' function to question1 and 2\"\n    data = data.dropna(how=\"any\")\n    data[\"question_text\"] = data[\"question_text\"].apply(clean_sentence)\n    \n    return data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e9f633b98e41ddbfc60f86ff6bf064601e33c02d"},"cell_type":"code","source":"data = clean_dataframe(data)\ndata.head(5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"9f7fea6f55e31965c3ddcd55221457be68170839"},"cell_type":"code","source":"def build_corpus(data):\n    \"Creates a list of lists containing words from each sentence\"\n    corpus = []\n    for sentence in data[\"question_text\"].iteritems():\n        word_list = sentence[1].split(\" \")\n        corpus.append(word_list)\n            \n    return corpus","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a759bbfb8849af538bb21046beb3e770e5d11fed"},"cell_type":"code","source":"corpus = build_corpus(data)        \ncorpus[0:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"aaedec0a2831915f36e3cd31b6dc6d839449901f"},"cell_type":"code","source":"def tsne_plot(model):\n    \"Creates and TSNE model and plots it\"\n    labels = []\n    tokens = []\n\n    for word in model.wv.vocab:\n        tokens.append(model[word])\n        labels.append(word)\n    \n    tsne_model = TSNE(perplexity=40, n_components=2, init='pca', n_iter=2500, random_state=23)\n    new_values = tsne_model.fit_transform(tokens)\n\n    x = []\n    y = []\n    for value in new_values:\n        x.append(value[0])\n        y.append(value[1])\n        \n    plt.figure(figsize=(16, 16)) \n    for i in range(len(x)):\n        plt.scatter(x[i],y[i])\n        plt.annotate(labels[i],\n                     xy=(x[i], y[i]),\n                     xytext=(5, 2),\n                     textcoords='offset points',\n                     ha='right',\n                     va='bottom')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"edc76d9314c96dc987fd7bf67101a847e6d2e022"},"cell_type":"code","source":"model = word2vec.Word2Vec(corpus, size=100, window=20, min_count=200, workers=4)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b68acdeb492ba54b202d62938edf0bd05c9e600e"},"cell_type":"code","source":"tsne_plot(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"dfd2d8c20280652e4c029ede9c22136f8ca135ab"},"cell_type":"code","source":"model = word2vec.Word2Vec(corpus, size=100, window=20, min_count=500, workers=4)\ntsne_plot(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"5de340bdb8c2005c96678bd36e7d86febe17aa88"},"cell_type":"code","source":"model = word2vec.Word2Vec(corpus, size=100, window=20, min_count=100, workers=4)\ntsne_plot(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"cee64191614ffaaaf7eead9b463fb0eb015ac2a9"},"cell_type":"code","source":"model.most_similar('phone')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"95e06d1fe67e280c3808cc969c2cdf666eafdf7c"},"cell_type":"code","source":"model.most_similar('trump')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85ef3c1118b4feae5516676f7e2f56d429e206f8"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}