{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q pyicu\n!pip install -q pycld2\n!pip install -q polyglot\n!pip install -q textstat\n!pip install -q googletrans\n!pip install morfessor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport gc\nimport re\nimport folium\nimport textstat\nfrom scipy import stats\nfrom colorama import Fore, Back, Style, init\n\nimport math\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\n\nimport random\nimport networkx as nx\nfrom pandas import Timestamp\n\nfrom PIL import Image\nfrom IPython.display import SVG\nfrom keras.utils import model_to_dot\n\nimport requests\nfrom IPython.display import HTML\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\n\ntqdm.pandas()\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\nfrom nltk.stem.wordnet import WordNetLemmatizer \nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer  \n\nimport nltk\nfrom textblob import TextBlob\n\nfrom nltk.corpus import wordnet\nfrom nltk.corpus import stopwords\nfrom googletrans import Translator\nfrom nltk import WordNetLemmatizer\nfrom polyglot.detect import Detector\nfrom nltk.stem import WordNetLemmatizer\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\n\nstopword=set(STOPWORDS)\n\nlem = WordNetLemmatizer()\ntokenizer=TweetTokenizer()\n\nnp.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\"\nos.listdir(DATA_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_PATH = DATA_PATH + \"test.csv\"\nVAL_PATH = DATA_PATH + \"validation.csv\"\nTRAIN_PATH = DATA_PATH + \"jigsaw-toxic-comment-train.csv\"\n\nval_data = pd.read_csv(VAL_PATH)\ntest_data = pd.read_csv(TEST_PATH)\ntrain_data = pd.read_csv(TRAIN_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Wordcloud of all comments","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\ntext = ' '.join([nonan(abstract) for abstract in train_data[\"comment_text\"]])\nwordcloud = WordCloud(max_font_size=None, background_color='black', collocations=False,\n                      width=1200, height=1000).generate(text)\nfig = px.imshow(wordcloud)\nfig.update_layout(title_text='Common words in comments')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Language distribution","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_language(text):\n    return Detector(\"\".join(x for x in text if x.isprintable()), quiet=True).languages[0].name\n\ntrain_data[\"lang\"] = train_data[\"comment_text\"].progress_apply(get_language)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lang_list = sorted(list(set(train_data[\"lang\"])))\ncounts = [list(train_data[\"lang\"]).count(cont) for cont in lang_list]\ndf = pd.DataFrame(np.transpose([lang_list, counts]))\ndf.columns = [\"Language\", \"Count\"]\ndf[\"Count\"] = df[\"Count\"].apply(int)\n\ndf_en = pd.DataFrame(np.transpose([[\"English\", \"Non-English\"], [max(counts), sum(counts) - max(counts)]]))\ndf_en.columns = [\"Language\", \"Count\"]\n\nfig = px.bar(df_en, x=\"Language\", y=\"Count\", title=\"Language of comments\", color=\"Language\", text=\"Count\")\nfig.update_layout(template=\"plotly_white\")\nfig.data[0].marker.line.color = 'rgb(0, 0, 0)'\nfig.data[0].marker.line.width = 0.5\nfig.data[1].marker.line.color = 'rgb(0, 0, 0)'\nfig.data[1].marker.line.width = 0.5\nfig.data[0].textfont.color = \"black\"\nfig.data[0].textposition = \"outside\"\nfig.data[1].textfont.color = \"black\"\nfig.data[1].textposition = \"outside\"\nfig","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Bar chart of non-English languages","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = px.bar(df.query(\"Language != 'en' and Language != 'un'\").query(\"Count >= 50\"),\n             y=\"Language\", x=\"Count\", title=\"Language of non-English comments\", template=\"plotly_white\", color=\"Language\", text=\"Count\", orientation=\"h\")\nfig.update_traces(marker=dict(line=dict(width=0.75,\n                                        color='black')),  textposition=\"outside\")\nfig.update_layout(showlegend=False)\nfig","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Pie chart of non-English languages","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = go.Figure([go.Pie(labels=df.query(\"Language != 'en' and Language != 'un'\").query(\"Count >= 50\")[\"Language\"],\n           values=df.query(\"Language != 'en' and Language != 'un'\").query(\"Count >= 50\")[\"Count\"])])\nfig.update_layout(title_text=\"Pie chart of non-English languages\", template=\"plotly_white\")\nfig.data[0].marker.colors = [px.colors.qualitative.Plotly[2:]]\nfig.data[0].textfont.color = \"black\"\nfig.data[0].textposition = \"outside\"\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Wordclouds for different categories","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Non-toxic vs. Toxic**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"clean_mask=np.array(Image.open(\"../input/imagesforkernal/safe-zone.png\"))\nclean_mask=clean_mask[:,:,1]\n\nsubset = train_data.query(\"toxic == 0\")\ntext = subset.comment_text.values\nwc = WordCloud(background_color=\"black\",max_words=2000,mask=clean_mask,stopwords=stopword)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(7.5, 7.5))\nplt.axis(\"off\")\nplt.title(\"Words frequented in Clean Comments\", fontsize=16)\nplt.imshow(wc.recolor(colormap= 'viridis' , random_state=17), alpha=0.98)\nplt.show()\n\nclean_mask=np.array(Image.open(\"../input/imagesforkernal/swords.png\"))\nclean_mask=clean_mask[:,:,1]\n\nsubset = train_data.query(\"toxic == 1\")\ntext = subset.comment_text.values\nwc = WordCloud(background_color=\"black\",max_words=2000,mask=clean_mask,stopwords=stopword)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(7.5, 7.5))\nplt.axis(\"off\")\nplt.title(\"Words frequented in Toxic Comments\", fontsize=16)\nplt.imshow(wc.recolor(colormap= 'viridis' , random_state=17), alpha=0.98)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Obscene vs. Severe Toxic vs. Threat vs. Insult**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic_mask=np.array(Image.open(\"../input/imagesforkernal/toxic-sign.png\"))\ntoxic_mask=toxic_mask[:,:,1]\n#wordcloud for clean comments\nsubset=train_data.query(\"obscene == 1\")\ntext=subset.comment_text.values\nwc= WordCloud(background_color=\"black\",max_words=4000,mask=toxic_mask,stopwords=stopword)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(20,20))\nplt.subplot(221)\nplt.axis(\"off\")\nplt.title(\"Words frequented in Obscene Comments\", fontsize=20)\nplt.imshow(wc.recolor(colormap= 'gist_earth' , random_state=244), alpha=0.98)\n\n#Severely toxic comments\nplt.subplot(222)\nsevere_toxic_mask=np.array(Image.open(\"../input/imagesforkernal/bomb.png\"))\nsevere_toxic_mask=severe_toxic_mask[:,:,1]\nsubset=train_data[train_data.severe_toxic==1]\ntext=subset.comment_text.values\nwc= WordCloud(background_color=\"black\",max_words=2000,mask=severe_toxic_mask,stopwords=stopword)\nwc.generate(\" \".join(text))\nplt.axis(\"off\")\nplt.title(\"Words frequented in Severe Toxic Comments\", fontsize=20)\nplt.imshow(wc.recolor(colormap= 'Reds' , random_state=244), alpha=0.98)\n\n#Threat comments\nplt.subplot(223)\nthreat_mask=np.array(Image.open(\"../input/imagesforkernal/anger.png\"))\nthreat_mask=threat_mask[:,:,1]\nsubset=train_data[train_data.threat==1]\ntext=subset.comment_text.values\nwc= WordCloud(background_color=\"black\",max_words=2000,mask=threat_mask,stopwords=stopword)\nwc.generate(\" \".join(text))\nplt.axis(\"off\")\nplt.title(\"Words frequented in Threatening Comments\", fontsize=20)\nplt.imshow(wc.recolor(colormap= 'summer' , random_state=2534), alpha=0.98)\n\n#insult\nplt.subplot(224)\ninsult_mask=np.array(Image.open(\"../input/imagesforkernal/swords.png\"))\ninsult_mask=insult_mask[:,:,1]\nsubset=train_data[train_data.insult==1]\ntext=subset.comment_text.values\nwc= WordCloud(background_color=\"black\",max_words=2000,mask=insult_mask,stopwords=stopword)\nwc.generate(\" \".join(text))\nplt.axis(\"off\")\nplt.title(\"Words frequented in insult Comments\", fontsize=20)\nplt.imshow(wc.recolor(colormap= 'Paired_r' , random_state=244), alpha=0.98)\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}