{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install -q langdetect\n!pip install -q textstat","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\n\npd.set_option('display.max_rows', 100)\npd.set_option('display.max_columns', 100)\npd.set_option('display.width', 1000)\n\nfrom collections import defaultdict,Counter\nfrom multiprocessing import Pool\n\nimport textstat\nfrom statistics import *\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nimport plotly.graph_objects as go\nfrom plotly.subplots import make_subplots\nimport plotly.figure_factory as ff\nimport plotly.express as px\nimport plotly.offline as py\n\nfrom sklearn.feature_extraction.text import CountVectorizer\nfrom nltk.stem import WordNetLemmatizer, PorterStemmer\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\n\nfrom langdetect import detect\n\nfrom wordcloud import WordCloud, STOPWORDS\n\nfrom scipy.stats import norm, kurtosis, skew\n\nfrom tqdm import tqdm\ntqdm.pandas() \nimport string, json, nltk, gc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stop = set(stopwords.words('english'))\nplt.style.use('seaborn')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_UNINTENDED_BIAS = \"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\"\nTRAIN_TOXICITY = \"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\"\n\nVALIDATION = \"../input/jigsaw-multilingual-toxic-comment-classification/validation.csv\"\n\nTEST = \"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1_df = pd.read_csv(TRAIN_UNINTENDED_BIAS)\ntrain_2_df = pd.read_csv(TRAIN_TOXICITY)\n\nvalidation_df = pd.read_csv(VALIDATION)\n\ntest_df = pd.read_csv(TEST)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_2_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 3, figsize=(20, 5))\n\nsns.countplot(train_1_df['toxic'].astype(int), ax=ax[0])\nax[0].set_title('Unintended bias dataset')\n\nsns.countplot(train_2_df['toxic'].astype(int), ax=ax[1])\nax[1].set_title('Toxicity dataset')\n\nsns.countplot(validation_df['toxic'].astype(int), ax=ax[2])\nax[2].set_title('Validation dataset')\n\nfig.suptitle('Toxicity distribution across datasets', fontweight='bold', fontsize=14)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15, 5))\n\nsns.countplot(validation_df['lang'], ax=ax[0])\nax[0].set_title('Validation')\n\nsns.countplot(test_df['lang'], ax=ax[1])\nax[1].set_title('Test')\n\nfig.suptitle('Language distribution across datasets', fontweight=\"bold\", fontsize=14)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(15, 5))\n\nsns.distplot(train_1_df[train_1_df['toxic']==0]['comment_text'].str.len(), axlabel=\"Non toxic\", ax=ax[0])\nsns.distplot(train_1_df[train_1_df['toxic']==1]['comment_text'].str.len(), axlabel=\"Toxic\", ax=ax[1])\n\nfig.show()\n\nfig.suptitle(\"Distribution of number of No: Characters in Comments - Unintended bias dataset\", fontsize=14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(15, 5))\n\nsns.distplot(train_2_df[train_2_df['toxic']==0]['comment_text'].str.len(), axlabel=\"Non toxic\", ax=ax[0])\nsns.distplot(train_2_df[train_2_df['toxic']==1]['comment_text'].str.len(), axlabel=\"Toxic\", ax=ax[1])\n\nfig.show()\n\nfig.suptitle(\"Distribution of number of No: Characters in Comments - Toxicity dataset\", fontsize=14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1_df[train_1_df['comment_text'].str.len() > 850][['comment_text', 'toxic']].sample(n=100).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_2_df[train_2_df['comment_text'].str.len() > 2000][[\"comment_text\", \"toxic\"]].sample(n=100).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,3, figsize=(15, 5))\n\nsns.distplot(validation_df[validation_df['toxic']==0]['comment_text'].str.len(), axlabel=\"Validation - Non toxic\", ax=ax[0])\nsns.distplot(validation_df[validation_df['toxic']==1]['comment_text'].str.len(), axlabel=\"Validation - Toxic\", ax=ax[1])\nsns.distplot(test_df['content'].str.len(), axlabel=\"Test\", ax=ax[2])\n\nfig.show()\n\nfig.suptitle(\"Distribution of number of No: Characters in Comments - Toxicity dataset\", fontsize=14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation_df[validation_df['comment_text'].str.len() > 1000][[\"comment_text\", 'lang', \"toxic\"]].sample(n=100).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_df[test_df['content'].str.len() > 1000][[\"content\", 'lang']].sample(n=100).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15, 5))\n\nvalidation_df[\"character_count\"] = validation_df['comment_text'].apply(lambda x: len(x))\ntest_df['character_count'] = test_df['content'].apply(lambda x: len(x))\n\ntest_df['character_count'] = test_df['character_count'].apply(lambda x: 1000 if x > 1000 else x) # Nicer formatting \n\nsns.boxplot('lang', 'character_count', data=validation_df, ax=ax[0])\nsns.boxplot('lang', 'character_count', data=test_df, ax=ax[1])\n\nfig.show()\n\nfig.suptitle('Distribution of # of characters for each language')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(15, 5))\n\nsns.distplot(train_1_df[train_1_df['toxic']==0]['comment_text'].str.split().str.len(), axlabel=\"Non toxic\", ax=ax[0])\nsns.distplot(train_1_df[train_1_df['toxic']==1]['comment_text'].str.split().str.len(), axlabel=\"Toxic\", ax=ax[1])\n\nfig.show()\n\nfig.suptitle(\"Distribution of number of No: Words in Comments - Unintended bias\", fontsize=14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2, figsize=(15, 5))\n\nsns.distplot(train_2_df[train_2_df['toxic']==0]['comment_text'].str.split().str.len(), axlabel=\"Non toxic\", ax=ax[0])\nsns.distplot(train_2_df[train_2_df['toxic']==1]['comment_text'].str.split().str.len(), axlabel=\"Toxic\", ax=ax[1])\n\nfig.show()\n\nfig.suptitle(\"Distribution of number of No: Words in Comments - Toxicity\", fontsize=14)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def whisker_plot_stats(train):\n    ## Number of words \n    train[\"num_words\"] = train[\"comment_text\"].progress_apply(lambda x: len(str(x).split()))\n\n    ## Number of unique words \n    train[\"num_unique_words\"] = train[\"comment_text\"].progress_apply(lambda x: len(set(str(x).split())))\n\n    ## Number of characters \n    train[\"num_chars\"] = train[\"comment_text\"].progress_apply(lambda x: len(str(x)))\n\n    ## Number of stopwords \n    train[\"num_stopwords\"] = train[\"comment_text\"].progress_apply(lambda x: len([w for w in str(x).lower().split() if w in STOPWORDS]))\n\n    ## Number of punctuations \n    train[\"num_punctuations\"] =train['comment_text'].progress_apply(lambda x: len([c for c in str(x) if c in string.punctuation]) )\n\n    ## Number of title case words\n    train[\"num_words_upper\"] = train[\"comment_text\"].progress_apply(lambda x: len([w for w in str(x).split() if w.isupper()]))\n\n    # Number of title case words\n    train[\"num_words_title\"] = train[\"comment_text\"].progress_apply(lambda x: len([w for w in str(x).split() if w.istitle()]))\n\n    # Average length of the words\n    train[\"mean_word_len\"] = train[\"comment_text\"].progress_apply(lambda x: np.mean([len(w) for w in str(x).split()]))\n    \n    return train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Train 1...')\ntrain_1_df = whisker_plot_stats(train_1_df)\nprint('Train 2...')\ntrain_2_df = whisker_plot_stats(train_2_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1_df['num_words'].loc[train_1_df['num_words']>100] = 100\ntrain_1_df['num_punctuations'].loc[train_1_df['num_punctuations']>10] = 10 \ntrain_1_df['num_chars'].loc[train_1_df['num_chars']>350] = 350 \ntrain_1_df['toxic'] = train_1_df['toxic'].apply(lambda x: 1 if x > 0.5 else 0)\n\ntrain_2_df['num_words'].loc[train_2_df['num_words']>100] = 100\ntrain_2_df['num_punctuations'].loc[train_2_df['num_punctuations']>10] = 10 \ntrain_2_df['num_chars'].loc[train_2_df['num_chars']>350] = 350 \n\n\n# figure related code\nfig, ax = plt.subplots(1, 2, figsize=(15, 5))\nfig.suptitle('Distribution of # words in toxicity dataset', fontsize=14, fontweight='bold')\n\nsns.boxplot(x='toxic', y='num_words', data=train_1_df, ax=ax[0])\nax[0].set_title('Unintended bias dataset')\n\nsns.boxplot(x='toxic', y='num_words', data=train_2_df, ax=ax[1])\nax[1].set_title('Toxicity dataset')\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_1_df[train_1_df['num_words'] >= 100]['comment_text'].sample(n=100).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_2_df[train_2_df['num_words'] >= 100]['comment_text'].sample(n=100).reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def preprocess_comments(df, stop=stop, n=1, col='comment_text'):\n    new_corpus=[]\n    \n    stem = PorterStemmer()\n    lem = WordNetLemmatizer()\n    \n    for text in tqdm(df[col], total=len(df)):\n        words = [w for w in word_tokenize(text) if (w not in stop)]\n       \n        words = [lem.lemmatize(w) for w in words if(len(w)>n)]\n     \n        new_corpus.append(words)\n        \n    new_corpus = [word for l in new_corpus for word in l]\n    \n    return new_corpus","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig,ax = plt.subplots(1, 2, figsize=(15,7))\n\nfor i in range(2):\n    new = train_1_df[train_1_df['toxic']== i]\n    corpus_train = preprocess_comments(new, {})\n    \n    dic = defaultdict(int)\n    for word in corpus_train:\n        if word in stop:\n            dic[word]+=1\n            \n    top = sorted(dic.items(), key=lambda x:x[1],reverse=True)[:10] \n    x, y = zip(*top)\n    ax[i].bar(x,y)\n    ax[i].set_title(str(i))\n\nfig.suptitle(\"Common stopwords in unintented bias dataset\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(20,12))\n\nfor i in range(2):\n    new = train_2_df[train_2_df['toxic']==i]   \n    corpus = corpus_train\n    counter = Counter(corpus)\n    most = counter.most_common()\n    x = []\n    y = []\n    \n    for word,count in most[:20]:\n        if (word not in stop) :\n            x.append(word)\n            y.append(count)\n            \n    sns.barplot(x=y, y=x, ax=ax[i])\n    ax[i].set_title(str(i))\n    \nfig.suptitle(\"Common words in toxicity dataset\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(20,12))\n\nfor i in range(2):\n    new = train_2_df[train_2_df['toxic']==i]   \n    corpus = corpus_train_2\n    counter = Counter(corpus)\n    most = counter.most_common()\n    x = []\n    y = []\n    \n    for word,count in most[:20]:\n        if (word not in stop) :\n            x.append(word)\n            y.append(count)\n            \n    sns.barplot(x=y, y=x, ax=ax[i])\n    ax[i].set_title(str(i))\n    \nfig.suptitle(\"Common words in unintended bias dataset\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_top_ngram(corpus, n=None):\n    vec = CountVectorizer(ngram_range=(n, n),stop_words=stop).fit(corpus)\n    bag_of_words = vec.transform(corpus)\n    sum_words = bag_of_words.sum(axis=0) \n    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]\n    words_freq = sorted(words_freq, key = lambda x: x[1], reverse=True)\n    return words_freq[:20]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1, 2, figsize=(15, 10))\n\nfor i in range(2):\n    new = train_1_df[train_1_df['toxic'] == i]['comment_text']\n    top_n_bigrams = get_top_ngram(new, 2)[:20]\n    x, y = map(list, zip(*top_n_bigrams))\n    sns.barplot(x=y, y=x, ax=ax[i])\n    ax[i].set_title(str(i))\n    \nfig.suptitle('Common bigrams in unintended bias dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(15,10))\n\nfor i in range(2):\n    new = train_2_df[train_2_df['toxic'] == i]['comment_text']\n    top_n_bigrams = get_top_ngram(new, 2)[:20]\n    x, y = map(list,zip(*top_n_bigrams))\n    sns.barplot(x=y,y=x,ax=ax[i])\n    ax[i].set_title(str(i))\n    \nfig.suptitle(\"Common bigrams in toxicity dataset\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = set(STOPWORDS)\n\ndef show_wordcloud(data, title = None,ax=None):\n    wordcloud = WordCloud(\n        background_color='white',\n        stopwords=stopwords,\n        max_words=100,\n        max_font_size=30, \n        scale=3,\n        random_state=1 \n        )\n    \n    wordcloud = wordcloud.generate(str(data))\n    ax.imshow(wordcloud,interpolation='nearest')\n    ax.axis('off')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(20,12))\n\nfor i in range(2):\n    new = train_2_df[train_2_df['toxic'] == i]['comment_text']\n    show_wordcloud(new, ax=ax[i])\n    ax[i].set_title(str(i))\n    \nfig.suptitle('Wordcloud for toxicity dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(1,2,figsize=(20,12))\n\nfor i in range(2):\n    new = train_1_df[train_1_df['toxic'] == i]['comment_text']\n    show_wordcloud(new, ax=ax[i])\n    ax[i].set_title(str(i))\n    \nfig.suptitle('Wordcloud for unintended bias dataset')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plot_readability(a, b, title, bins=0.4):\n    \n    # Setting limits\n    a = a[a >= 0]\n    a = a[a <= 100]\n    b = b[b >= 0]\n    b = b[b <= 100]\n    \n    trace1 = ff.create_distplot([a, b], ['non toxic', 'toxic'], bin_size=bins, show_rug=False)\n    trace1['layout'].update(title=title)\n    \n    py.iplot(trace1, filename='Distplot')\n    \n    table_data= [[\"Statistical Measures\",\"non toxic\",'toxic'],\n                 [\"Mean\",mean(a),mean(b)],\n                 [\"Standard Deviation\",pstdev(a),pstdev(b)],\n                 [\"Variance\",pvariance(a),pvariance(b)],\n                 [\"Median\",median(a),median(b)],\n                 [\"Maximum value\",max(a),max(b)],\n                 [\"Minimum value\",min(a),min(b)]]\n    \n    trace2 = ff.create_table(table_data)\n    py.iplot(trace2, filename='Table')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fre_non_toxic = np.array(train_1_df[\"comment_text\"][train_1_df[\"toxic\"].astype(int) == 0].sample(n=150000).apply(textstat.flesch_reading_ease))\nfre_toxic = np.array(train_1_df[\"comment_text\"][train_1_df[\"toxic\"].astype(int) == 1].apply(textstat.flesch_reading_ease))\n\nplot_readability(fre_non_toxic, fre_toxic, \"Flesch Reading Ease - Unintended bias dataset\", 1) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fre_non_toxic = np.array(train_2_df['comment_text'][train_2_df['toxic'].astype(int) == 0].apply(textstat.flesch_reading_ease))\nfre_toxic = np.array(train_2_df['comment_text'][train_2_df['toxic'].astype(int) == 1].apply(textstat.flesch_reading_ease))\n\nplot_readability(fre_non_toxic, fre_toxic, \"Flesch Reading Ease - Toxicity dataset\", 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fre_non_toxic = np.array(validation_df['comment_text'][validation_df['toxic'].astype(int) == 0].apply(textstat.flesch_reading_ease))\nfre_toxic = np.array(validation_df['comment_text'][validation_df['toxic'].astype(int) == 1].apply(textstat.flesch_reading_ease))\n\nplot_readability(fre_non_toxic, fre_toxic, \"Flesch Reading Ease - Validation set\", 1)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}