{"cells":[{"metadata":{"_uuid":"2dbbff1c74fba6f73f4182830171206c324915b6"},"cell_type":"markdown","source":"# Quora EDA\n\n1. Embeddings\n2. Open trainset and testset\n3. N-gram analysis\n4. Hyperparameters\n5. References"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np \nimport pandas as pd\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"840823a08129998ba6ddc592f6e8e59aee0e2bac"},"cell_type":"code","source":"!ls ../input/\n\nprint(\"\\nEmbeddings:\")\n!ls ../input/embeddings/","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ef138e7bedeeee43de5dc4e129141a4f81cb5b61"},"cell_type":"markdown","source":"## 1. Embeddings\n\n* GoogleNews-vectors-negative300 - https://code.google.com/archive/p/word2vec/\n* glove.840B.300d - https://nlp.stanford.edu/projects/glove/\n* paragram_300_sl999 - https://cogcomp.org/page/resource_view/106\n* wiki-news-300d-1M - https://fasttext.cc/docs/en/english-vectors.html"},{"metadata":{"trusted":true,"_uuid":"577e8d3b18c3b4da618571995f96196651b6f9de"},"cell_type":"code","source":"print('File sizes')\nfor f in os.listdir('../input'):\n    if 'zip' not in f:\n        print(f.ljust(30) + str(round(os.path.getsize('../input/' + f) / 1000000, 2)) + 'MB')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e5d945c93793073b68cb4b77ed6da5815ea0baf0"},"cell_type":"markdown","source":"## 2. Open trainset and testset"},{"metadata":{"trusted":true,"_uuid":"866b5d46465241b6a005abadfbf75bc3b200a536"},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv').fillna(' ')\ntest = pd.read_csv('../input/test.csv').fillna(' ')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"91131a1a244a09431e0277071855507885f8dae3"},"cell_type":"code","source":"print(\"Shape of training set: \", train.shape)\nprint(\"Shape of test set: \", test.shape)\n\ntrain_target = train['target'].values\nnp.unique(train_target)\nprint(\"\\nPercentage of insincere questions irt sincere questions: \", train_target.mean(), \"%\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"6fbf6edf551d9306e200de105e07c7cde8698e7c"},"cell_type":"code","source":"train.sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"ce4749543a5d2ff25e62aebfae1b798fb71da3f2"},"cell_type":"code","source":"test.sample(10)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a1bf6db5561456aff8964e54954b438b2e75635a"},"cell_type":"code","source":"insincere_q = train[train[\"target\"] == 1][\"question_text\"].tolist()\n\nwith open('insinceres.txt', 'w') as f:\n    for item in insincere_q:\n        f.write(\"%s\\n\" % item)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c88df5e3870b430000f142be5335ef44fb5c0bc6"},"cell_type":"markdown","source":"## 3. N-gram analysis"},{"metadata":{"trusted":true,"_uuid":"f8977442c6e6b52d5c11e781b324b7d6f87e3892","_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"from collections import defaultdict\nfrom nltk.corpus import stopwords\nfrom plotly import tools\nimport plotly.offline as py\npy.init_notebook_mode(connected=True)\nimport plotly.graph_objs as go\n\nstop_words = set(stopwords.words('english')) \ninsinc_df = train[train.target==1]\nsinc_df = train[train.target==0]\n\ndef plot_ngrams(n_grams):\n\n    ## custom function for ngram generation ##\n    def generate_ngrams(text, n_gram=1):\n        token = [token for token in text.lower().split(\" \") if token != \"\" if token not in stop_words]\n        ngrams = zip(*[token[i:] for i in range(n_gram)])\n        return [\" \".join(ngram) for ngram in ngrams]\n\n    ## custom function for horizontal bar chart ##\n    def horizontal_bar_chart(df, color):\n        trace = go.Bar(\n            y=df[\"word\"].values[::-1],\n            x=df[\"wordcount\"].values[::-1],\n            showlegend=False,\n            orientation = 'h',\n            marker=dict(\n                color=color,\n            ),\n        )\n        return trace\n\n    def get_bar(df, bar_color):\n        freq_dict = defaultdict(int)\n        for sent in df[\"question_text\"]:\n            for word in generate_ngrams(sent, n_grams):\n                freq_dict[word] += 1\n        fd_sorted = pd.DataFrame(sorted(freq_dict.items(), key=lambda x: x[1])[::-1])\n        fd_sorted.columns = [\"word\", \"wordcount\"]\n        trace = horizontal_bar_chart(fd_sorted.head(10), bar_color)\n        return trace    \n\n    trace0 = get_bar(sinc_df, 'blue')\n    trace1 = get_bar(insinc_df, 'blue')\n\n    # Creating two subplots\n    if n_grams == 1:\n        wrd = \"words\"\n    elif n_grams == 2:\n        wrd = \"bigrams\"\n    elif n_grams == 3:\n        wrd = \"trigrams\"\n    \n    fig = tools.make_subplots(rows=1, cols=2, vertical_spacing=0.04,\n                              subplot_titles=[\"Frequent \" + wrd + \" of sincere questions\", \n                                              \"Frequent \" + wrd + \" of insincere questions\"])\n    fig.append_trace(trace0, 1, 1)\n    fig.append_trace(trace1, 1, 2)\n    fig['layout'].update(height=500, width=1150, paper_bgcolor='rgb(233,233,233)', title=wrd + \" Count Plots\")\n    py.iplot(fig, filename='word-plots')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"11aefca40f1be0894aff64cd432f7c948514d225"},"cell_type":"code","source":"plot_ngrams(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"a611fa3718b5d0ad341d7fef4e9d185730474659"},"cell_type":"code","source":"plot_ngrams(2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"3bd837725f0c58e64a8fa719f9591298586b9e9f"},"cell_type":"code","source":"plot_ngrams(3)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a59972a4211268ae224282d4a41c257ed9f7dd3b"},"cell_type":"markdown","source":"## 4. Hyperparameters"},{"metadata":{"trusted":true,"_uuid":"6642bd8fdf0eb3709f5ffad14fbc3fb696608788"},"cell_type":"code","source":"## Number of words in the text\ntrain[\"num_words\"] = train[\"question_text\"].apply(lambda x: len(str(x).split()))\ntest[\"num_words\"] = test[\"question_text\"].apply(lambda x: len(str(x).split()))\n\n## Number of unique words in the text\ntrain[\"num_unique_words\"] = train[\"question_text\"].apply(lambda x: len(set(str(x).split())))\ntest[\"num_unique_words\"] = test[\"question_text\"].apply(lambda x: len(set(str(x).split())))\n\n## Number of characters in the text\ntrain[\"num_chars\"] = train[\"question_text\"].apply(lambda x: len(str(x)))\ntest[\"num_chars\"] = test[\"question_text\"].apply(lambda x: len(str(x)))\n\n## Number of stopwords in the text\ntrain[\"num_stopwords\"] = train[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in stop_words]))\ntest[\"num_stopwords\"] = test[\"question_text\"].apply(lambda x: len([w for w in str(x).lower().split() if w in stop_words]))\n\n## Average length of the words in the text\ntrain[\"mean_word_len\"] = train[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))\ntest[\"mean_word_len\"] = test[\"question_text\"].apply(lambda x: np.mean([len(w) for w in str(x).split()]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b24de38863c9793f99cc64a5846d205be8a2abc9"},"cell_type":"code","source":"## Truncate some extreme values for better visuals ##\ntrain['num_words'].loc[train['num_words']>50] = 50\ntrain['num_unique_words'].loc[train['num_unique_words']>50] = 50\ntrain['num_chars'].loc[train['num_chars']>300] = 300\ntrain['mean_word_len'].loc[train['mean_word_len']>10] = 10\n\nf, axes = plt.subplots(5, 1, figsize=(15,40))\n\nsns.boxplot(x='target', y='num_words', data=train, ax=axes[0])\naxes[0].set_xlabel('Target', fontsize=12)\naxes[0].set_title(\"Number of words in each class\", fontsize=15)\n\nsns.boxplot(x='target', y='num_unique_words', data=train, ax=axes[1])\naxes[1].set_xlabel('Target', fontsize=12)\naxes[1].set_title(\"Number of unique words in each class\", fontsize=15)\n\nsns.boxplot(x='target', y='num_chars', data=train, ax=axes[2])\naxes[2].set_xlabel('Target', fontsize=12)\naxes[2].set_title(\"Number of characters in each class\", fontsize=15)\n\nsns.boxplot(x='target', y='num_stopwords', data=train, ax=axes[3])\naxes[3].set_xlabel('Target', fontsize=12)\naxes[3].set_title(\"Number of stopwords in each class\", fontsize=15)\n\nsns.boxplot(x='target', y='mean_word_len', data=train, ax=axes[4])\naxes[4].set_xlabel('Target', fontsize=12)\naxes[4].set_title(\"Mean word length in each class\", fontsize=15)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"_uuid":"3fa7c2b5998f9f0297a05fde5fedfa04455b6e68"},"cell_type":"code","source":"print(train.columns)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"649d2da9ae84fd04d07872e944ffa88a5815115f"},"cell_type":"markdown","source":"## 5. References\n\n* [General EDA](https://www.kaggle.com/tunguz/just-some-simple-eda)\n* [Exploration notebook](https://www.kaggle.com/sudalairajkumar/simple-exploration-notebook-qiqc)"},{"metadata":{"trusted":true,"_uuid":"298aa079bab58fccd3cdd8d25d40e206ea33d92c"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}