{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\nimport os\nprint(os.listdir(\"../input\"))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/train.csv')\ntest = pd.read_csv('../input/test.csv')\n\ntrain_char = pd.read_csv('../input/train.csv')\ntest_char = pd.read_csv('../input/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\n#nltk.download('popular')\n\nimport re\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer\n\n# text cleaning & tokenization\ndef tokenize(text):\n    \n    # clean text\n    text = text.encode('ascii', 'ignore').decode('ascii')\n    text = text.lower()\n    \n    text = re.sub(r'\\b(?:(?:https?|ftp)://)?\\w[\\w-]*(?:\\.[\\w-]+)+\\S*', ' ', text) # remove hyperlink,subs charact in the brackets\n    text = re.sub(\"[\\r\\n]\", ' ', text) # remove new line characters\n    #text = re.sub(r'[^\\w\\s]','',text)\n    text = text.strip() ## convert to lowercase split indv words\n    \n    #tokens = word_tokenize(text)\n    # use TweetTokenizer instead of word_tokenize -> to prevent splitting at apostrophies\n    tknzr = TweetTokenizer()\n    tokens = tknzr.tokenize(text)\n    \n    # retain tokens with at least two words\n    tokens = [token for token in tokens if re.match(r'.*[a-z]{2,}.*', token)]\n    \n    # remove stopwords - optional\n    # removing stopwords lost important information\n    #if stop_set != None:\n    stop_set = stopwords.words('english')\n    tokens = [token for token in tokens if token not in stop_set]\n    \n    # lemmmatization - optional\n    #if lemmatizer != None:\n    lemmatizer = WordNetLemmatizer() \n    tokens = [lemmatizer.lemmatize(token) for token in tokens]\n    return tokens\n\ntrain['tokens'] = train['question_text'].map(lambda x: tokenize(x))\ntest['tokens'] = test['question_text'].map(lambda x: tokenize(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t1 = train[['qid','tokens','question_text']]\nt2 = t1.append(test)\nt2.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#######################################\n#     Co-occurance words (bigram)     #\n#######################################\nimport pandas as pd\nimport numpy as np\nimport nltk\nfrom nltk.corpus import stopwords\nimport gensim\nfrom gensim.models import LdaModel\nfrom gensim import models, corpora, similarities\nimport re\nfrom nltk.stem.porter import PorterStemmer\nimport time\nfrom nltk import FreqDist\nfrom scipy.stats import entropy\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nsns.set_style(\"darkgrid\")\nimport nltk\nfrom nltk import bigrams\nfrom nltk.corpus import stopwords\nimport re\nimport networkx as nx\nimport warnings\nimport matplotlib.pyplot as plt\nimport itertools\nimport collections\nimport nltk\nfrom nltk import bigrams\nt3 = t2[['tokens']]\nt4 = t3['tokens'].tolist()\nterms_bigram = [list(bigrams(tweet)) for tweet in t4]\n# View bigrams for the first tweet\n#https://www.earthdatascience.org/courses/earth-analytics-python/using-apis-natural-language-processing-twitter/calculate-tweet-word-bigrams-networks-in-python/\nterms_bigram[0]\nbigrams = list(itertools.chain(*terms_bigram))\n\n# Create counter of words in clean bigrams\nbigram_counts = collections.Counter(bigrams)\n\nbigram_counts.most_common(30)\n\nbigram_df = pd.DataFrame(bigram_counts.most_common(30),\n                             columns=['bigram', 'count'])\n\nbigram_df\n\nd = bigram_df.set_index('bigram').T.to_dict('records')\n# Create network plot \nG = nx.Graph()\n\n# Create connections between nodes\nfor k, v in d[0].items():\n    G.add_edge(k[0], k[1], weight=(v * 10))\n\n#G.add_node(\"dummy\", weight=100)\nfig, ax = plt.subplots(figsize=(25, 20))\n\npos = nx.spring_layout(G, k=16)\n\n# Plot networks\nnx.draw_networkx(G, pos,\n                 font_size=16,\n                 width=3,\n                 edge_color='grey',\n                 node_color='purple',\n                 with_labels = False,\n                 ax=ax)\n\n# Create offset labels\nfor key, value in pos.items():\n    x, y = value[0]+.135, value[1]+.045\n    ax.text(x, y,\n            s=key,\n            bbox=dict(facecolor='red', alpha=0.25),\n            horizontalalignment='center', fontsize=30)\n    \nplt.show()\n##This plot displays the networks of co-occurring words in tweets\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"## plot bigram df\ntop_30 = bigram_df.head(20).sort_values(by=['count'])\ntop_30\nax = top_30.plot.barh(x='bigram', y='count', rot=0, color=(0.2, 0.4, 0.6, 0.6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"t5 = t2[['tokens']]\nt5 = t5['tokens'].tolist()\nt6  = list(itertools.chain(*t5))\n\n#######################################\n#        Most common Word             #\n#######################################\n\nfdist = FreqDist(t6)\ntop20_tweetcount = fdist.most_common(20)\ntop20_tweetcount = sorted(top20_tweetcount , key=lambda x: x[1])\nlist1 = []\nlist2 = []\nfor i in top20_tweetcount:\n   list1.append(i[0])\n   list2.append(i[1])\n\nprint(top20_tweetcount)\nlist\nef = pd.DataFrame({'Vocubulary':list1, 'Most common word':list2})\nbx = ef.plot.barh(x='Vocubulary', y='Most common word', rot=0, color=(0.2, 0.4, 0.6, 0.6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def tokenize_character(text):\n    \n    # clean text\n    text = text.encode('ascii', 'ignore').decode('ascii')\n    text = text.lower()\n    \n    text = re.sub(r'\\b(?:(?:https?|ftp)://)?\\w[\\w-]*(?:\\.[\\w-]+)+\\S*', ' ', text) # remove hyperlink,subs charact in the brackets\n    text = re.sub(\"[\\r\\n]\", ' ', text) # remove new line characters\n    #text = re.sub(r'[^\\w\\s]','',text)\n    text = text.strip() ## convert to lowercase split indv words\n    \n    #tokens = word_tokenize(text)\n    # use TweetTokenizer instead of word_tokenize -> to prevent splitting at apostrophies\n    tknzr = TweetTokenizer()\n    tokens = tknzr.tokenize(text)\n    \n    # retain tokens with at least two words\n    tokens = [token for token in tokens if re.match(r'.*[a-z]{2,}.*', token)]\n    \n    return tokens\n\ntrain_char['tokens'] = train_char['question_text'].map(lambda x: tokenize_character(x))\ntest_char['tokens'] = test_char['question_text'].map(lambda x: tokenize_character(x))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_char\np1 = train_char\np1 = p1[[\"target\", \"tokens\"]]\np1[\"word_count\"] = p1.tokens.apply(lambda x: len(x))\np1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def score_to_numeric(x):\n    if x==0:\n        return \"Insincere\"\n    if x==1:\n        return \"Sincere\"\np1['target_class'] = p1['target'].apply(score_to_numeric)\np1.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns\nsns.set(style=\"whitegrid\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ax = sns.boxplot(x=\"target_class\", y=\"word_count\", data=p1,                  \n                 order=[\"Insincere\", \"Sincere\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p1_zero = p1[p1[\"target\"]==0]\np1_zero = p1_zero[[\"tokens\", \"word_count\"]]\n\np1_one = p1[p1[\"target\"]==1]\np1_one = p1_one[[\"tokens\", \"word_count\"]]\n\np1_zero.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p2_zero = p1_zero[['tokens']]\np2_zero = p2_zero['tokens'].tolist()\np2_zero  = list(itertools.chain(*p2_zero))\n\nstop_set = stopwords.words('english')\np2_zero = [token for token in p2_zero if token not in stop_set]\n\nfdist = FreqDist(p2_zero)\ntop20_tweetcount = fdist.most_common(20)\ntop20_tweetcount = sorted(top20_tweetcount , key=lambda x: x[1])\nlist1 = []\nlist2 = []\nfor i in top20_tweetcount:\n   list1.append(i[0])\n   list2.append(i[1])\n\nprint(top20_tweetcount)\nlist\nef = pd.DataFrame({'Insincere Vocubulary':list1, 'Most common word':list2})\nbx = ef.plot.barh(x='Insincere Vocubulary', y='Most common word', rot=0, color=(0.2, 0.4, 0.6, 0.6))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"p2_one = p1_one[['tokens']]\np2_one = p2_one['tokens'].tolist()\np2_one  = list(itertools.chain(*p2_one))\n\nstop_set = stopwords.words('english')\np2_one = [token for token in p2_one if token not in stop_set]\n\nfdist = FreqDist(p2_one)\ntop20_tweetcount = fdist.most_common(20)\ntop20_tweetcount = sorted(top20_tweetcount , key=lambda x: x[1])\nlist1 = []\nlist2 = []\nfor i in top20_tweetcount:\n   list1.append(i[0])\n   list2.append(i[1])\n\nprint(top20_tweetcount)\nlist\nef = pd.DataFrame({'Sincere Vocubulary':list1, 'Most common word':list2})\nbx = ef.plot.barh(x='Sincere Vocubulary', y='Most common word', rot=0, color=(0.2, 0.4, 0.6, 0.6))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":1}