{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"scrolled":false},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true,"scrolled":false},"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\"\nos.listdir(DATA_PATH)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"TEST_PATH = DATA_PATH + \"test.csv\"\nVAL_PATH = DATA_PATH + \"validation.csv\"\nTRAIN_PATH = DATA_PATH + \"jigsaw-toxic-comment-train.csv\"\n\nval_data = pd.read_csv(VAL_PATH)\ntest_data = pd.read_csv(TEST_PATH)\ntrain_data = pd.read_csv(TRAIN_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"val_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Wordcloud - For training, val and test data","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"from wordcloud import WordCloud, STOPWORDS\nimport matplotlib.pyplot as plt \n\ndef clean_comment(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \" \")\n    else:\n        return \"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"text = train_data.apply(lambda row: clean_comment(row[\"comment_text\"]), axis=1).to_string()\nwordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,\n                      width=1200, height=1200, stopwords=set(STOPWORDS)).generate(text)\nplt.figure(figsize = (7, 7), facecolor = None) \nplt.imshow(wordcloud) \nplt.axis(\"off\") \nplt.tight_layout(pad = 0) \nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"text = val_data.apply(lambda row: clean_comment(row[\"comment_text\"]), axis=1).to_string()\nwordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,\n                      width=1200, height=1200, stopwords=set(STOPWORDS)).generate(text)\nplt.figure(figsize = (7, 7), facecolor = None) \nplt.imshow(wordcloud) \nplt.axis(\"off\") \nplt.tight_layout(pad = 0) \nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"text = test_data.apply(lambda row: clean_comment(row[\"content\"]), axis=1).to_string()\nwordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,\n                      width=1200, height=1200, stopwords=set(STOPWORDS)).generate(text)\nplt.figure(figsize = (7, 7), facecolor = None) \nplt.imshow(wordcloud) \nplt.axis(\"off\") \nplt.tight_layout(pad = 0) \nplt.show() ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Findings \n\n1. Test and validation data doesn't seem to be really similar to training data. \n2. Training data shows a lot english words, but the validation and test data shows many non-english data. \n3. This is going to an issue. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# About non-english comments","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"!apt-get install -y python-numpy libicu-dev\n\n!pip install pyicu==2.3.1\n\n!pip install morfessor\n\n!pip install pycld2\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"from polyglot.detect import Detector\nfrom polyglot.detect.langids import isoLangs\n\ndef detect_lang(comment):\n    return Detector(\"\".join(x for x in comment if x.isprintable()), quiet=True).languages[0].name\n    \ndef get_lang_name(code):\n    try:\n        return isoLangs[code][\"name\"]\n    except:\n        return \"Unknown\" ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"train_data[\"lang_code\"] = train_data.apply(lambda row: detect_lang(row[\"comment_text\"]), axis=1)\ntrain_data[\"lang_name\"] = train_data[\"lang_code\"].apply(lambda row: get_lang_name(row))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"print(\"\\n------ Training data -------\\n\")\nprint(\"-> Unknown language code: \", train_data[\"lang_code\"].loc[\n    train_data[\"lang_name\"] == \"Unknown\"].unique())\nprint(\"-> Unknown language examples count: \",train_data[\"lang_code\"].loc[\n    train_data[\"lang_name\"] == \"Unknown\"].count() )\nprint(\"-> Unknown language value counts: \\n\", train_data[\"lang_code\"].loc[\n    train_data[\"lang_name\"] == \"Unknown\"].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"print(\"\\n------ Validation data -------\\n\")\n  \nval_data[\"lang_name\"] = val_data[\"lang\"].apply(lambda row: get_lang_name(row))\nprint(\"-> Unknown language code: \", val_data[\"lang\"].loc[\n    val_data[\"lang_name\"] == \"Unknown\"].unique())\nprint(\"-> Unknown language examples count: \",val_data[\"lang\"].loc[\n    val_data[\"lang_name\"] == \"Unknown\"].count() )\nprint(\"-> Unknown language value counts: \\n\", val_data[\"lang\"].loc[\n    val_data[\"lang_name\"] == \"Unknown\"].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"print(\"\\n------ Testing data -------\\n\")\n  \ntest_data[\"lang_name\"] = test_data[\"lang\"].apply(lambda row: get_lang_name(row))\nprint(\"-> Unknown language code: \", test_data[\"lang\"].loc[\n    test_data[\"lang_name\"] == \"Unknown\"].unique())\nprint(\"-> Unknown language examples count: \",test_data[\"lang\"].loc[\n    test_data[\"lang_name\"] == \"Unknown\"].count() )\nprint(\"-> Unknown language value counts: \\n\", test_data[\"lang\"].loc[\n    test_data[\"lang_name\"] == \"Unknown\"].value_counts())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Findings \n\n1. There are 717 examples in the data whose language is unknown.\n2. Among them, 616 are completely unknown (Language code: 'un')\n3. Rest of them are not categorized by the polyglot package. ('jw', 'hmn', 'haw', 'nso', 'zu', 'zzp', 'tlh', 'kha', 'iw', 'mfe', 'crs', 'syr', 'xx')\n3. Interestingly, testing and validation data has no such instances of an unknown language. \n4. Thus the unknown language comments can be removed from training data. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# English Language v/s Non-English languages","execution_count":null},{"metadata":{"trusted":true,"scrolled":false},"cell_type":"code","source":"train_eng_count = train_data[\"lang_name\"].loc[train_data[\"lang_name\"] == \"English\"].count()\ntrain_non_eng_count = train_data[\"lang_name\"].loc[~train_data[\"lang_name\"].isin([\"English\", \"Unknown\"])].count()\n\nval_eng_count = val_data[\"lang_name\"].loc[val_data[\"lang_name\"] == \"English\"].count()\nval_non_eng_count = val_data[\"lang_name\"].loc[~val_data[\"lang_name\"].isin([\"English\", \"Unknown\"])].count()\n\ntest_eng_count = test_data[\"lang_name\"].loc[test_data[\"lang_name\"] == \"English\"].count()\ntest_non_eng_count = test_data[\"lang_name\"].loc[~test_data[\"lang_name\"].isin([\"English\", \"Unknown\"])].count()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"x = ['English', 'Non-English']\nlang_count = [[train_eng_count, train_non_eng_count], [val_eng_count, val_non_eng_count], \n          [test_eng_count, test_non_eng_count]]\nlang_count_titles =  [ \"Training data\", \"Validation data\", \"Testing data\"]\nx_pos = np.arange(len(x))\n\nfig, ax = plt.subplots(1, 3, figsize=(15,5))\n\nfor i, col in enumerate(ax):\n    tmp_data = lang_count[i]\n    col.bar(x_pos, tmp_data)\n#     plt.bar(x_pos, tmp_data)\n    for j, v in enumerate(tmp_data):\n        col.text(x_pos[j] - 0.1, v + .01, \"{:.2f}%\".format(v*100/sum(tmp_data)))\n    col.set_title(lang_count_titles[i])\n    col.set_xlabel(\"Language\")\n    col.set_ylabel(\"Count\")\n    col.set_xticks(x_pos)\n    col.set_xticklabels(x)\n    col.plot()\n\n\nfig.suptitle(\"Language Count Across Datasets\", fontsize = 33)\nfig.tight_layout()\nfig.subplots_adjust(top=0.8)\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Findings \n\n1. The data is a highly skewed. 99% of the comments are in English.\n2. But the validation and test data contains only non-english comments. \n3. So viable solution is to translate these comments to english and build a model on it.\n4. On validation and testing, first translate the comment to English and pass it to the model for prediction. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Major non-english languages","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train_non_eng_count = train_data[\"lang_name\"][~train_data[\"lang_name\"].isin(\n    ['English', 'Unknown'])].value_counts().rename_axis('lang').reset_index(name='lang_count')\n\nval_non_eng_count = val_data[\"lang_name\"][~val_data[\"lang_name\"].isin(\n    ['English', 'Unknown'])].value_counts().rename_axis('lang').reset_index(name='lang_count')\n\ntest_non_eng_count = test_data[\"lang_name\"][~test_data[\"lang_name\"].isin(\n    ['English', 'Unknown'])].value_counts().rename_axis('lang').reset_index(name='lang_count')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"df_set = [train_non_eng_count, val_non_eng_count, test_non_eng_count]\nnon_eng_count_titles =  [ \"Training data\", \"Validation data\", \"Testing data\"]\n\nfig, ax = plt.subplots(1, 3, figsize=(18,5))\n\nfor i, col in enumerate(ax):\n    tmp_data = df_set[i]\n    \n    if i == 0:\n        mean_count = tmp_data.lang_count.mean()\n        idx = tmp_data[tmp_data['lang_count'] > int(mean_count)].lang\n        vals = tmp_data[tmp_data['lang_count'] > int(mean_count)].lang_count\n    else:\n        idx = tmp_data.lang\n        vals = tmp_data.lang_count\n    col.pie(vals, labels=idx, autopct='%1.1f%%')\n    col.axis('equal')\n    col.set_title(non_eng_count_titles[i])\n    col.plot()\n\nfig.suptitle(\"Non English Count Across Datasets\", fontsize = 33)\nfig.tight_layout()\nfig.subplots_adjust(top=0.8)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Findings\n\n**Training data**\n1. The top-5 non-english languages are German, Scot, Danish, Arabic and Spanish\n2. They constitutes 34% of total contribution. \n3. Considered the languages with counts more than mean count, which is 17. \n\n**Validation Data**\n\n1. Didn't adhere mean count criteria since there are only a few languages present in the data. \n2. There are only three languages present in validation dataset.\n    - Turkish\n    - Spanish\n    - Italian\n \n**Testing Data**\n\n1. Didn't adhere mean count criteria since there are only a few languages present in the data. \n2. There are only six languages present in validation dataset.\n    - Turkish\n    - Spanish\n    - Italian\n    - French\n    - Portugese\n    - Russian","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Geographical distribution of comment data\n\nEnglish is a universal language. How other languages are distributed in the data geographically? ","execution_count":null},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"country_map = {\n    'Unknown': 'Unknown',\n    'English': 'United Kingdom',\n    'Azerbaijani': 'Azerbaijan',\n    'Basque': 'Spain',\n    'Interlingua': 'Italy',\n    'German': 'Germany',\n    'Volapük': 'Germany',\n    'Norwegian Nynorsk': 'Norway',\n    'Scots': 'Scotland',\n    'Dutch': 'Netherlands',\n    'Polish': 'Poland',\n    'Greek, Modern': 'Greece',\n    'Manx': 'Ireland',\n    'Portuguese': 'Portugal',\n    'Luganda': 'Uganda',\n    'Hungarian': 'Hungary',\n    'Kinyarwanda': 'Rwanda',\n    'Danish': 'Denmark',\n    'Latin': 'Italy',\n    'Western Frisian': 'Netherlands',\n    'Galician': 'Spain',\n    'Italian': 'Italy',\n    'Fijian': 'Fiji',\n    'Sanskrit (Saṁskṛta)': 'India',\n    'Occitan': 'Spain',\n    'Xhosa': 'South Africa',\n    'Quechua': 'Peru',\n    'Welsh': 'Wales',\n    'Maltese': 'Malta',\n    'Chichewa; Chewa; Nyanja': 'Zimbabwe',\n    'Czech': 'Czech Republic',\n    'Oromo': 'Ethiopia',\n    'Tagalog': 'Philippines',\n    'Romansh': 'Switzerland',\n    'Turkish': 'Turkey',\n    'Irish': 'Ireland',\n    'Venda': '\tSouth Africa',\n    'Indonesian': 'Indonesia',\n    'Icelandic': 'Iceland',\n    'Uzbek': 'Uzbekistan',\n    'Interlingue': 'Italy',\n    'Malagasy': 'Madagascar',\n    'Swedish': 'Sweden',\n    'Breton': 'France',\n    'Somali': 'Somalia',\n    'Luxembourgish, Letzeburgesch': 'Luxembourg',\n    'Swahili': 'Kenya',\n    'Faroese': 'Denmark',\n    'Wolof': 'Senegal',\n    'Spanish; Castilian': 'Spain',\n    'Tsonga': 'Mozambique',\n    'Shona': 'Zambia',\n    'French': 'France',\n    'Tonga (Tonga Islands)': 'Tonga Islands',\n    'Southern Sotho': 'South Africa',\n    'Norwegian': 'Norway',\n    'Lithuanian': 'Lithuania',\n    'Malay': 'Malaysia',\n    'Tamil': 'India',\n    'Corsican': 'France',\n    'Japanese': 'Japan',\n    'Turkmen': 'Turkmenistan',\n    'Scottish Gaelic; Gaelic': 'Scotland',\n    'Nauru': 'Nauru',\n    'Samoan': 'Samoa',\n    'Estonian': 'Estonia',\n    'Waray-Waray': 'Philippines',\n    'Latvian': 'Latvia',\n    'Albanian': 'Albania',\n    'Slovak': 'Slovakia',\n    'Haitian; Haitian Creole': 'Haiti',\n    'Esperanto': 'Brazil',\n    'Māori': 'New Zealand',\n    'Bulgarian': 'Bulgaria',\n    'Sundanese': 'Indonesia',\n    'Finnish': 'Finland',\n    'Tatar': 'Russia',\n    'Afar': 'Ethiopia',\n    'Romanian, Moldavian, Moldovan': 'Romania',\n    'Chinese': 'China',\n    'Tswana': 'South Africa',\n    'Zhuang, Chuang': '',\n    'Serbian': 'Serbia',\n    'Cebuano': 'China',\n    'Lingala': 'Democratic Republic of the Congo',\n    'Catalan; Valencian': 'Spain',\n    'Ukrainian': 'Ukraine',\n    'Persian': 'Iran',\n    'Marathi (Marāṭhī)': 'India',\n    'Guaraní': 'Paraguay',\n    'Korean': 'South Korea',\n    'Arabic': 'UAE',\n    'Bosnian': 'Bosnia',\n    'Vietnamese': 'Vietnam',\n    'Urdu': 'India',\n    'Thai': 'Thailand',\n    'Croatian': 'Croatia',\n    'Bengali': 'India',\n    'Kurdish': 'Iraq',\n    'Malayalam': 'India',\n    'Hindi': 'India',\n    'Macedonian': 'Macedonia',\n    'Aymara': 'Bolivia',\n    'Afrikaans': 'Australia',\n    'Georgian': 'Georgia',\n    'Oriya': 'India',\n    'Kannada': 'India',\n    'Russian': 'Russia',\n    'Tibetan Standard, Tibetan, Central': 'Tibet',\n    'Gujarati': 'India',\n    'Mongolian': 'Mangolia',\n    'Khmer': 'Vietnam',\n    'Kirundi': 'Tanzania',\n    'Nepali': 'Nepal',\n    'Sinhala, Sinhalese': '',\n    'Burmese': 'Burma',\n    'Kalaallisut, Greenlandic': '',\n    'Panjabi, Punjabi': 'India',\n    'Swati': 'South Africa',\n    'Yoruba': 'Nigeria',\n    'Kazakh': 'Kazakhstan',\n    'Hausa': 'Nigeria',\n    'Slovene': 'Slovenia',\n    'Tigrinya': 'Ethiopia',\n    'Pashto, Pushto': 'Pakistan',\n    'Akan': 'Ghana',\n    'Telugu': 'India',\n    'Bislama': 'Republic of Vanuatu',\n    'Igbo': 'Nigeria',\n    'Belarusian': 'Belarus'\n}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"train_data[\"country\"] = train_data.apply(lambda row: country_map[row[\"lang_name\"]], axis=1)\nval_data[\"country\"] = val_data.apply(lambda row: country_map[row[\"lang_name\"]], axis=1)\ntest_data[\"country\"] = test_data.apply(lambda row: country_map[row[\"lang_name\"]], axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true},"cell_type":"code","source":"import plotly.express as px","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"country_data_train = train_data.country.value_counts().rename_axis('country').reset_index(name='count')\nfig = px.choropleth(country_data_train.query(\"country != 'United Kingdom' and country != 'Unknown'\"), locations=\"country\", hover_name=\"country\",\n                     projection=\"natural earth\", locationmode=\"country names\", title=\"Training data - geographical distribution\", color=\"count\",\n                     template=\"plotly\", color_continuous_scale=\"agsunset\")\nfig.data[0].marker.line.color = 'rgb(0, 0, 0)'\nfig.data[0].marker.line.width = 0.2\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"country_data_val = val_data.country.value_counts().rename_axis('country').reset_index(name='count')\nfig = px.choropleth(country_data_val.query(\"country != 'United Kingdom' and country != 'Unknown'\"), locations=\"country\", hover_name=\"country\",\n                     projection=\"natural earth\", locationmode=\"country names\", title=\"Validation data - geographical distribution\", color=\"count\",\n                     template=\"plotly\", color_continuous_scale=\"agsunset\")\nfig.data[0].marker.line.color = 'rgb(0, 0, 0)'\nfig.data[0].marker.line.width = 0.2\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"country_data_test = test_data.country.value_counts().rename_axis('country').reset_index(name='count')\nfig = px.choropleth(country_data_test.query(\"country != 'United Kingdom' and country != 'Unknown'\"), locations=\"country\", hover_name=\"country\",\n                     projection=\"natural earth\", locationmode=\"country names\", title=\"Test data - geographical distribution\", color=\"count\",\n                     template=\"plotly\", color_continuous_scale=\"agsunset\")\nfig.data[0].marker.line.color = 'rgb(0, 0, 0)'\nfig.data[0].marker.line.width = 0.2\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Findings \n\n\n1. The english language was not considered since it is a universal language.\n2. That explains the absence or feeble presence of training data at USA, Canada etc. \n3. Also, the languages are allotted to majority speaking country. Eg. French is allotted only to France\n\n**Training data**\n\n1. According to the plot, comment data is coming from various countries all over the planet.\n2. In Asia, India has a significant representation in the data.\n3. Considering the population of China, surprisingly there is no instance of Mandarin. (But it could be in the list of unknown languages.)\n\n**Validation data**\n\n1. The validation data confines to only three countries\n    - Turkey\n    - Italy\n    - Spain\n    \n**Test data**\n\n1. The test data confines to only six countries\n    - Turkey\n    - Italy\n    - Spain\n    - Russia\n    - Portugal\n    - France","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Understanding categories of training & validation data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import seaborn as sns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data['label'] = 0\ntrain_data.loc[(train_data.toxic == 1) | (train_data.obscene == 1) | \n           (train_data.insult == 1) | (train_data.identity_hate == 1) | \n           (train_data.severe_toxic == 1) | (train_data.threat == 1), 'label'] = 1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7,7), )\nax = sns.countplot(x=\"label\", data=train_data)\nfor i, p in enumerate(ax.patches):\n    height = p.get_height()\n    ax.text(p.get_x() + p.get_width()/2., height + 0.7,\n        \"{:.2f}%\".format(train_data['label'].value_counts(normalize=True)[i]*100),\n            ha=\"center\", rotation=10)\nax.set_xticklabels(['Non-Toxic', 'Toxic'])\nax.set_xlabel(\"Comments\")\nax.set_ylabel(\"Count\")\nax.set_title('Toxic and Non-Toxic comments in training data', fontsize=14)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7,7), )\nax = sns.countplot(x=\"toxic\", data=val_data)\nfor i, p in enumerate(ax.patches):\n    height = p.get_height()\n    ax.text(p.get_x() + p.get_width()/2., height + 0.7,\n        \"{:.2f}%\".format(val_data['toxic'].value_counts(normalize=True)[i]*100),\n            ha=\"center\", rotation=10)\nax.set_xticklabels(['Non-Toxic', 'Toxic'])\nax.set_xlabel(\"Comments\")\nax.set_ylabel(\"Count\")\nax.set_title('Toxic and Non-Toxic comments in validation data', fontsize=14)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Findings\n\n1. Training data is heavily skewed. 90% of data is non-toxic. \n2. Validation data follows the same pattern. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Understanding categories with in toxic comments","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"comment_cat_df = pd.melt(train_data[['toxic','severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nfig, ax = plt.subplots(nrows=1, ncols=1, figsize=(7,7), )\nax = sns.countplot(data=comment_cat_df[comment_cat_df.value == 1], \n                   x='variable', \n                  order=comment_cat_df[comment_cat_df.value == 1].variable.value_counts().index)\nfor i, p in enumerate(ax.patches):\n    height = p.get_height()\n    ax.text(p.get_x() + p.get_width()/2., height + 0.7,\n        \"{:.2f}%\".format(comment_cat_df[comment_cat_df.value==1].variable.value_counts(normalize=True)[i]*100),\n            ha=\"center\", rotation=10)\n    ax.set_xticklabels(ax.get_xticklabels(), rotation=10, ha=\"right\")\n    ax.set_xlabel(\"Toxic comments\")\n    ax.set_ylabel(\"Count\")\n\n\nax.set_title('Categories within toxic comments', fontsize=14)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Findings\n\n1. Among the toxic comments, 43% is coming under `toxic` category. \n2. `obscene` and `toxic` categories overlaps the most.  \n3. `threat` category messages are the least in the training set. ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Save the updated datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.to_csv(\"notebook_one_train_data.csv\", index = False)\nval_data.to_csv(\"notebook_one_val_data.csv\", index = False)\ntest_data.to_csv(\"notebook_one_test_data.csv\", index = False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}