{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nimport warnings\n\nwarnings.filterwarnings('ignore')\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\ntrain_df = train.copy()\ntest_df = test.copy()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"51b50d6fcb559dddef1e6c3c52e2dc5a666ca2ce"},"cell_type":"code","source":"(train.columns,test.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"22d9dbe1cc1e350defab7d7ecfefbd5158816062"},"cell_type":"code","source":"train.sample()\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"06946ca0df64d3a2665df057e33d039f1999c18c"},"cell_type":"code","source":"target_counts = train['target'].value_counts()\nprint(\"Sincere questions:%i (%.1f%%)\" %(target_counts[0],float(100*target_counts[0]/(target_counts[0]+target_counts[1]))))\nprint(\"Insincere questions: %i (%.1f%%)\" %(target_counts[1],float(100*target_counts[1]/(target_counts[0]+target_counts[1]))))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"b1fe101488e5f5d87caec090c2379c09c4a050ab"},"cell_type":"code","source":"sns.countplot(train['target'], hue=train[\"target\"])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"62b5cf6e6144477a55a38c57fb6572a06dd483f7"},"cell_type":"code","source":"train['target'].value_counts().plot.bar()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"e4e0ccd4a8faababb784688d604e55215c88e279"},"cell_type":"code","source":"fig1, ax1 = plt.subplots()\nax1.pie(train['target'].value_counts().values, explode=(0,0.1),labels=['Sincere','Insincere'], autopct='%1.1f%%',\n        shadow=True, startangle=130)\nax1.axis('equal')  # Equal aspect ratio ensures that pie is drawn as a circle.\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"0ef98256d07b160d817cb75f41fa2df6e6d9548a"},"cell_type":"code","source":"train['target'].value_counts().values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"7c8684ef6b612562cad451e5d4666c22ac347768"},"cell_type":"code","source":"\"\"\"\n#Example for Ngraming using nltk\nfrom nltk import ngrams\n\nsentence = 'I am Dr. Santhosh'\n\nn = 2\nsixgrams = ngrams(sentence.split(), n)\n\nfor grams in sixgrams:\n  print(grams)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"85c1b3dc4beb8b791661c86096ff368411cb6ba2"},"cell_type":"code","source":"\"\"\"\n#Example For Ngraming method without using nltk\n#sentence = 'this is a foo bar sentences and i want to ngramize it'\nsentence = \"I am Dr. Santhosh\"\ntokens = sentence.split(\" \")\nsequence = [tokens[i:] for i in range(3)]\nonegram = zip(*sequence)\n#print(sequence)\n#print(onegram)\ngramList = [\" \".join(ngram) for ngram in onegram]\nprint(gramList)\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8ace653b1a5a4e183a9d4aff7901c51d7b4b830d"},"cell_type":"markdown","source":"#train['target']==1 - This returns the boolean value for the row indexes based on the condition. This is useful for splitting data"},{"metadata":{"trusted":true,"_uuid":"d6dffa9ca7a71dde7497d25ed7134d53055bfb5a"},"cell_type":"code","source":"from nltk import ngrams\nfrom wordcloud import WordCloud, STOPWORDS\nfrom collections import defaultdict\n\nsincere_text = train[train['target']==1]\ninsincere_text = train[train['target']==0]\ndef generate_ngrams(sentence, n):\n    tokens = [token for token in sentence.lower().split(\" \") if token != \"\" if token not in STOPWORDS]\n    return [\" \".join(grams) for grams in ngrams(tokens, n)]\ndef get_ngram_count(df,n):\n    freq_dict = defaultdict(int)\n    for text in df[\"question_text\"]:\n        for word in generate_ngrams(text,n):\n            freq_dict[word] += 1\n            \n    return freq_dict\nprint(get_ngram_count(insincere_text,3))\n#generate_ngrams(\"I am Dr. Santhosh\", 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_uuid":"2f2ecfa380b3b324053a371cb839709b17eb7a21"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.6","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}