{"cells":[{"metadata":{"_uuid":"9b92f9dd-f63f-4350-835a-ed8daa0dc627","_cell_guid":"e7d317a1-d57c-414a-b47d-08ba47f9250a","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd\nimport tensorflow as tf# data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a5e1c93-c43d-42cc-8043-487dda4b74cc","_cell_guid":"d27f7b65-6737-4d20-a5ad-d1bc5216d452","trusted":true},"cell_type":"code","source":"!pip install -q pyicu\n!pip install -q pycld2\n!pip install -q polyglot\n!pip install -q textstat\n!pip install -q googletrans","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"49c8fb48-60f0-439d-b869-4841d1ab0674","_cell_guid":"275df0d4-81e0-47f4-84aa-c4334fbdc69f","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport os\nimport gc\nimport re\nimport folium\nimport textstat\nfrom scipy import stats\nfrom colorama import Fore, Back, Style, init\n\nimport math\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\n\nimport random\nimport networkx as nx\nfrom pandas import Timestamp\n\nfrom PIL import Image\nfrom IPython.display import SVG\nfrom keras.utils import model_to_dot\n\nimport requests\nfrom IPython.display import HTML\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\n\ntqdm.pandas()\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\nimport transformers\nimport tensorflow as tf\n\nfrom tensorflow.keras.callbacks import Callback\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\n\nfrom tensorflow.keras.models import Model\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.optimizers import Adam\nfrom tokenizers import BertWordPieceTokenizer\nfrom tensorflow.keras.layers import Dense, Input, Dropout, Embedding\nfrom tensorflow.keras.layers import LSTM, GRU, Conv1D, SpatialDropout1D\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import activations\nfrom tensorflow.keras import constraints\nfrom tensorflow.keras import initializers\nfrom tensorflow.keras import regularizers\n\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.activations import *\nfrom tensorflow.keras.constraints import *\nfrom tensorflow.keras.initializers import *\nfrom tensorflow.keras.regularizers import *\n\nfrom sklearn import metrics\nfrom sklearn.utils import shuffle\nfrom gensim.models import Word2Vec\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_extraction.text import TfidfVectorizer,\\\n                                            CountVectorizer,\\\n                                            HashingVectorizer\n\nfrom nltk.stem.wordnet import WordNetLemmatizer \nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer  \n\nimport nltk\nfrom textblob import TextBlob\n\nfrom nltk.corpus import wordnet\nfrom nltk.corpus import stopwords\nfrom googletrans import Translator\nfrom nltk import WordNetLemmatizer\nfrom polyglot.detect import Detector\nfrom nltk.stem import WordNetLemmatizer\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\n\nstopword=set(STOPWORDS)\n\nlem = WordNetLemmatizer()\ntokenizer=TweetTokenizer()\n\nnp.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bbbff600-5bcf-417e-bfcf-20f52744fd30","_cell_guid":"eee5650b-880b-4bf3-bc5d-9e9f5b5f8d4d","trusted":true},"cell_type":"markdown","source":"Importing twython this is an important Twitter package for comment analysis"},{"metadata":{"_uuid":"91ac537c-04a7-40d1-8cff-8555a8ea51ea","_cell_guid":"c061268b-6ad3-42f9-84ff-ccae4affddfc","trusted":true},"cell_type":"code","source":"#importing important packages\n! pip install -q twython\n\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ab9b9573-4f8b-4fdc-9ab6-2fdef9c90df1","_cell_guid":"22f8cfea-a8d1-4030-b04b-6d6d9f1d5adf","trusted":true},"cell_type":"code","source":"\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2ef29d50-2ced-4fd5-aba1-88eef17e5e0f","_cell_guid":"8122c141-7795-499c-aa76-295140c96776","trusted":true},"cell_type":"code","source":"train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\ntrain2.toxic = train2.toxic.round().astype(int)\n\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ff9822e0-73f4-4445-acbe-3fab0548808b","_cell_guid":"b4b0da71-5380-4186-8e4b-81527331a877","trusted":true},"cell_type":"code","source":"train1.tail()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b852e933-fe02-4e7a-905d-05822942a4cd","_cell_guid":"98cc9d73-5ba3-4088-b873-1dbfe82d54e3","trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b0e21368-6f71-4d74-b924-909e66b44c36","_cell_guid":"83da5c85-fa18-43f8-8cef-9812bc3add1b","trusted":true},"cell_type":"markdown","source":"**LET'S LOOK AT THE DIVISION**"},{"metadata":{"_uuid":"5e1fca65-ad8b-4be4-8000-956b9a881a73","_cell_guid":"15485e5d-2b41-4dd8-8147-fd880ef785df","trusted":true},"cell_type":"code","source":"print('TRAIN::',train1.shape[0],'TEST::',test.shape[0])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"690e7a02-a527-4e7c-801d-e14a626aeae4","_cell_guid":"21f39fb0-55ed-4792-ae09-799d4d9f302e","trusted":true},"cell_type":"markdown","source":"**FINDING THE MOST COMMON WORDS**"},{"metadata":{"_uuid":"79472470-1743-45ea-b797-f1cac1fd4181","_cell_guid":"c774ab6c-a503-4f5c-b31b-4f3d8c538bc4","trusted":true},"cell_type":"code","source":"from wordcloud import WordCloud\nimport plotly.express as px\nimport matplotlib.pyplot as plt\n\ndef remove_nan(word):\n    if type(word)==str:\n        return word.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\n#replacing nan with ''\ntext = ' '.join([remove_nan(x) for x in train1['comment_text']])\n\nwordcloud = WordCloud(max_font_size = None,background_color = 'red',collocations=False,width =1500,height = 1200).generate(text)\nfig = px.imshow(wordcloud)\nfig.update_layout(title_text = ' Most Common words in comments')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d9d04959-6310-4e3e-a62d-d487c97fc3d4","_cell_guid":"1f651bed-ea4b-4287-a2ed-9acce540fd76","trusted":true},"cell_type":"markdown","source":"**MOST COMMON WORDS WE SEE HERE ARE-** article, page,talk, will,one , will, edit* ,*obscence words are less often"},{"metadata":{"_uuid":"35b3a400-abee-49d6-89ff-57d64b1a8788","_cell_guid":"4802efa2-4dc3-4bc0-9626-0c3cb57fb1e7","trusted":true},"cell_type":"markdown","source":"**Dealing With Null values**"},{"metadata":{"_uuid":"b0f7e839-c53f-4e6c-ba66-8a14cd5c1b37","_cell_guid":"6d708675-5863-4b8f-8ee7-ff19afbf0ab0","trusted":true},"cell_type":"code","source":"print('Train Data Null values:')\ncheck_null = train1.isnull().sum()\nprint(check_null)\nprint('Test Data Null values:')\ncheck_null = test.isnull().sum()\nprint(check_null)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"36ccbe5b-0f57-4c38-971a-ac839fca1ca4","_cell_guid":"5ca60b98-6a84-42d8-bd9d-2f506c3e3e1c","trusted":true},"cell_type":"markdown","source":"**There are no null values the data is clean so far :)**"},{"metadata":{"_uuid":"5d45928a-b139-4684-a1be-d8351bc2283b","_cell_guid":"d849782c-f72a-449c-b5ea-b03b51790352","trusted":true},"cell_type":"markdown","source":"**Crrating a bar plot for different negative comments category**"},{"metadata":{"_uuid":"c0479293-36b6-4340-8754-d01b1755d566","_cell_guid":"e35b795e-697f-441d-8d30-bef79d0e566f","trusted":true},"cell_type":"code","source":"\nimport seaborn as sns\ntarget_columns = train1.iloc[:,2:].sum()\nplt.figure(figsize=(16,8))\nax = sns.barplot(target_columns.index, target_columns.values,alpha=0.9 )\nplt.title(\" Division per category\")\nplt.xlabel(\"Category\")\nplt.ylabel(\"Frequency\")\n\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ac642a49-68d6-4166-9d5d-9db5698e5b4b","_cell_guid":"1b50963c-10e9-4c4d-ab76-197fca3f302f","trusted":true},"cell_type":"code","source":"#Checking  the class imbalance\nx=train1.iloc[:,2:].sum()\nrowsums = train1.iloc[:,2:].sum(axis=1)\ntrain1['clean']= (rowsums==0)\n#Checking how many comments are totally clean without any negative tags\ntrain1['clean'].sum()\nprint('Total comments are  =  ',len(train1))\nprint('Total clean comments = ',train1['clean'].sum())\nprint('Total number of tags',x.sum())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9fe458be-b5ce-48b9-b084-1dfa1f527825","_cell_guid":"002c0162-0a90-4d8e-9d44-ef8bf7b297b3","trusted":true},"cell_type":"markdown","source":"**Checking for multiple tags**"},{"metadata":{"_uuid":"694cbe51-149f-41b0-8ac0-eafd508d54c5","_cell_guid":"673ad5c7-f498-45ab-99f5-3d4601f091cf","trusted":true},"cell_type":"code","source":"train_data=train1\nx_data=train_data.iloc[:,2:].sum(axis=1).value_counts()\n#plot\nimport plotly.express as px\nfig = px.bar(x_data, x=x_data.index, y=x_data.values)\nfig.update_layout(title_text=\"Multiple tags\", template=\"plotly_white\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cb2a149a-aa07-4765-94b8-3541e588721c","_cell_guid":"c33275e7-689b-4ed4-8f57-1742af13e7a2","trusted":true},"cell_type":"markdown","source":"**LET'S GENERATE SOME MORE NASTY COMMENTS** FOR FUN !!"},{"metadata":{"_uuid":"12cb89dc-e873-4fef-b116-3e63e14d7ed6","_cell_guid":"7e87e779-b26e-4c8e-b608-64d007c38f97","trusted":true},"cell_type":"code","source":"train_data=train1\nimport markovify as mk\ndoc = train_data.loc[train_data.clean==0,'comment_text'].tolist()\ntext_model = mk.Text(doc)\nfor i in range(5):\n    print(text_model.make_sentence())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83d3f677-6404-4247-9850-7da3a769cb4a","_cell_guid":"b5c7d284-c151-4cf9-80c6-00fc44989e38","trusted":true},"cell_type":"markdown","source":"**Now let's create word clouds of different target tag categories i.e Obscene, Toxic, Identity Threat etc**"},{"metadata":{"_uuid":"d1ddb0b4-954a-46fb-b3ae-6b82fdbfa47d","_cell_guid":"61406709-e499-421c-877c-528211cbd868","trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom nltk.corpus import stopwords\nstop_words=set(stopwords.words())\nclean_mask = np.array(Image.open(\"../input/images/safe-zone.png\"))\nclean_mask = clean_mask[:,:,1]\n#wordcloud for clean comments\nsubset = train_data[train_data.clean==1]\ntext = subset.comment_text.values\nwc = WordCloud(background_color='black',max_words=2000,mask=clean_mask,stopwords=stop_words)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(20,10))\nplt.axis('off')\nplt.title('Words frequent in clean comments',fontsize=20)\nplt.imshow(wc.recolor(colormap = 'viridis',random_state=17), alpha=0.98)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e055a874-2d24-4773-8e00-6f603e2a329e","_cell_guid":"f8a99545-304d-4e96-955b-e5e7560a2e7c","trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom nltk.corpus import stopwords\nstop_words=set(stopwords.words())\nclean_mask = np.array(Image.open(\"../input/images/nuclear.png\"))\nclean_mask = clean_mask[:,:,1]\n#wordcloud for clean comments\nsubset = train_data[train_data.toxic==1]\ntext = subset.comment_text.values\nwc = WordCloud(background_color='black',max_words=2000,mask=clean_mask,stopwords=stop_words)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(20,10))\nplt.axis('off')\nplt.title(' Frequency of words in Toxic comments',fontsize=20)\nplt.imshow(wc.recolor(colormap = 'viridis',random_state=17), alpha=0.98)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"086c8d9f-da13-453c-9706-d508c358e2a3","_cell_guid":"05a41fc5-f50e-4a4e-a7fb-fa7ea509696f","trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom nltk.corpus import stopwords\nstop_words=set(stopwords.words())\nclean_mask = np.array(Image.open(\"../input/images/nuclear.png\"))\nclean_mask = clean_mask[:,:,1]\n#wordcloud for clean comments\nsubset = train_data[train_data.severe_toxic==1]\ntext = subset.comment_text.values\nwc = WordCloud(background_color='white',max_words=2000,mask=clean_mask,stopwords=stop_words)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(20,10))\nplt.axis('off')\nplt.title('Words frequent in Severe comments',fontsize=20)\nplt.imshow(wc.recolor(colormap = 'viridis',random_state=17), alpha=0.98)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"804a901e-5fe2-49da-bae8-3d42d36c1fd2","_cell_guid":"6ae98e02-6223-46be-91dd-a31bf0e1c956","trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom nltk.corpus import stopwords\nstop_words=set(stopwords.words())\nclean_mask = np.array(Image.open(\"../input/images/nuclear.png\"))\nclean_mask = clean_mask[:,:,1]\n#wordcloud for clean comments\nsubset = train_data[train_data.obscene==1]\ntext = subset.comment_text.values\nwc = WordCloud(background_color='red',max_words=2000,mask=clean_mask,stopwords=stop_words)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(20,10))\nplt.axis('off')\nplt.title('Words frequent in obscene comments',fontsize=20)\nplt.imshow(wc.recolor(colormap = 'viridis',random_state=17), alpha=1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b41b6905-df73-4afa-ba44-1a014e7c2aaf","_cell_guid":"9f552bc0-2f2b-4bf1-8ed6-db759468b3f3","trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom nltk.corpus import stopwords\nstop_words=set(stopwords.words())\nclean_mask = np.array(Image.open(\"../input/images/nuclear.png\"))\nclean_mask = clean_mask[:,:,1]\n#wordcloud for clean comments\nsubset = train_data[train_data.insult==1]\ntext = subset.comment_text.values\nwc = WordCloud(background_color='black',max_words=2000,mask=clean_mask,stopwords=stop_words)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(20,10))\nplt.axis('off')\nplt.title('Words frequent in insult comments',fontsize=20)\nplt.imshow(wc.recolor(colormap = 'viridis',random_state=17), alpha=1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bd39b483-694c-4e8d-83a0-266c2754de2a","_cell_guid":"18199bf0-e236-4f95-a356-8de72b0ff03a","trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom nltk.corpus import stopwords\nstop_words=set(stopwords.words())\nclean_mask = np.array(Image.open(\"../input/images/nuclear.png\"))\nclean_mask = clean_mask[:,:,1]\n#wordcloud for clean comments\nsubset = train_data[train_data.threat==1]\ntext = subset.comment_text.values\nwc = WordCloud(background_color='pink',max_words=2000,mask=clean_mask,stopwords=stop_words)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(20,10))\nplt.axis('off')\nplt.title('Words frequent in threat comments',fontsize=20)\nplt.imshow(wc.recolor(colormap = 'viridis',random_state=17), alpha=1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7b545d26-23fa-4095-975b-4b9391d555ec","_cell_guid":"bc1c1598-d711-4866-ba49-80d403298c68","trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom nltk.corpus import stopwords\nstop_words=set(stopwords.words())\nclean_mask = np.array(Image.open(\"../input/images/nuclear.png\"))\nclean_mask = clean_mask[:,:,1]\n#wordcloud for clean comments\nsubset = train_data[train_data.insult==1]\ntext = subset.comment_text.values\nwc = WordCloud(background_color='blue',max_words=2000,mask=clean_mask,stopwords=stop_words)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(20,10))\nplt.axis('off')\nplt.title('Words frequent in insult comments',fontsize=20)\nplt.imshow(wc.recolor(colormap = 'viridis',random_state=17), alpha=0.98)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2459746d-eafb-41d4-8a3c-9453815bce12","_cell_guid":"1934db4d-ce29-4227-a018-677f4c413071","trusted":true},"cell_type":"code","source":"from PIL import Image\nfrom nltk.corpus import stopwords\nstop_words=set(stopwords.words())\nclean_mask = np.array(Image.open(\"../input/images/nuclear.png\"))\nclean_mask = clean_mask[:,:,1]\n#wordcloud for clean comments\nsubset = train_data[train_data.identity_hate==1]\ntext = subset.comment_text.values\nwc = WordCloud(background_color='black',max_words=2000,mask=clean_mask,stopwords=stop_words)\nwc.generate(\" \".join(text))\nplt.figure(figsize=(20,10))\nplt.axis('off')\nplt.title('Words frequent in identity hate comments',fontsize=20)\nplt.imshow(wc.recolor(colormap = 'viridis',random_state=17), alpha=1)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"83762fb6-9678-4485-b4bc-a3b6740f0d05","_cell_guid":"148825a4-d5c4-4e1b-a309-9fee07cfbb27","trusted":true},"cell_type":"code","source":"\ntest_data=test\nval_data=valid\nfig,axes = plt.subplots(ncols=3,figsize=(17,7),dpi=100)\ntemp = val_data['lang'].value_counts()\nsns.barplot(temp.index,temp,ax=axes[0],palette='Set1')\n\ntemp = test_data['lang'].value_counts()\nsns.barplot(temp.index,temp,ax=axes[1],palette='Set1')\nsns.countplot(data=val_data , x='lang' , hue='toxic',ax=axes[2],palette='Set1')\naxes[0].set_ylabel('Count')\naxes[1].set_ylabel(' ')\naxes[2].set_ylabel(' ')\naxes[2].set_xlabel(' ')\naxes[0].set_title('Language distribution of Validation Dataset',fontsize=13)\naxes[1].set_title('Language distribution of Test Dataset', fontsize=13)\naxes[2].set_title('Language distribution by Target of Validation Dataset',fontsize=13)\nplt.tight_layout()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1ddc135-fcdf-4ba6-be38-075aebb908fb","_cell_guid":"90e5e590-8cdf-4dfa-ab13-43b380d86ec2","trusted":true},"cell_type":"markdown","source":"# Feature Engineering"},{"metadata":{"_uuid":"ec3c22a5-205b-49ae-927e-7576838fcc69","_cell_guid":"8eee32b3-cbba-4b4d-a778-b1100db0b5f7","trusted":true},"cell_type":"markdown","source":"**Plot of word density per comment.**"},{"metadata":{"_uuid":"dfda4041-a529-4c75-8c15-9817be378d6d","_cell_guid":"7df86faa-715c-4673-bbd8-1ab2c622b6a0","trusted":true},"cell_type":"code","source":"def word_counter(x):\n    if type(x) is str:\n        return len(x.split())\n    else:\n        return 0\n    \ntrain_data['comment_word'] = train_data['comment_text'].apply(word_counter)\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n\n# matplotlib histogram\nplt.hist(train_data['comment_word'], color = 'blue', edgecolor = 'black',\n         bins = int(180/5))\n\n# seaborn histogram\nsns.distplot(train_data['comment_word'], hist=True, kde=False, \n             bins=int(180/5), color = 'blue',\n             hist_kws={'edgecolor':'black'})\n# Add labels\nplt.title('word density distribution')\nplt.xlabel('word count')\nplt.ylabel('')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"6770aa84-af4a-4a29-89b6-1cd13efcef2e","_cell_guid":"9f7bf7ee-83f5-4913-a1bc-60ffe20a4768","trusted":true},"cell_type":"markdown","source":"# Sentiment Analysis"},{"metadata":{"_uuid":"279e0035-6a4f-4951-a134-30eda0ba909a","_cell_guid":"ef9f95ab-051e-4a26-83ca-ea5667ad933a","trusted":true},"cell_type":"markdown","source":"**Generally comments with negative sentiments tend to be more toxic**"},{"metadata":{"_uuid":"8f750fb4-da9f-43f7-b86f-264f4174c941","_cell_guid":"61cf3bcd-3055-4899-9186-8a64ff935abc","trusted":true},"cell_type":"code","source":"from tqdm import tqdm, tqdm_notebook\ntqdm_notebook().pandas()\ndef polarity(x):\n    if type(x) == str:\n        return SIA.polarity_scores(x)\n    else:\n        return 1000\n    \nSIA = SentimentIntensityAnalyzer()\ntrain_data[\"polarity\"] = train_data[\"comment_text\"].progress_apply(polarity)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"008085ee-31f8-4644-beb9-11d7f15f7269","_cell_guid":"50c8e395-f465-455a-973f-41f7faeb1c09","trusted":true},"cell_type":"code","source":"import plotly.graph_objects as go\n\nfig = go.FigureWidget(go.Histogram(x=[pols[\"neg\"] for pols in train_data[\"polarity\"] if pols[\"neg\"] != 0], marker=dict(\n            color='skyblue')\n    ))\n\nfig.update_layout(xaxis_title=\"Negativity level\", title_text=\"Negativity sentiment\", template=\"simple_white\")\nfig","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"190a2028-d7ca-4e80-9f2e-a47d4357266e","_cell_guid":"c2b0abee-5911-46b5-b342-d71cc103bf43","trusted":true},"cell_type":"code","source":"val = val_data\ntrain = train_data\n\ndef clean(text):\n    text = text.fillna(\"fillna\").str.lower()\n    text = text.map(lambda x: re.sub('\\\\n',' ',str(x)))\n    text = text.map(lambda x: re.sub(\"\\[\\[User.*\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n    return text\n\nval[\"comment_text\"] = clean(val[\"comment_text\"])\ntest_data[\"content\"] = clean(test_data[\"content\"])\ntrain[\"comment_text\"] = clean(train[\"comment_text\"])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0bfbebb7-3b60-4594-94f1-1d99f84cb9ac","_cell_guid":"2441c5b6-5ef3-44c5-86de-37bb9d9fa6ae","trusted":true},"cell_type":"code","source":"# Modelling","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a2758e8e-27dc-47a6-9221-f9dcca4ef466","_cell_guid":"74f1f0a8-e618-47f1-a486-85617e93712a","trusted":true},"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids=[]\n    \n    for i in range(0, len(texts), chunk_size):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n        \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"232c2cf6-75e3-41e7-8f86-9cfceafe6520","_cell_guid":"dfb65c64-7f9f-4b86-94ed-e31f153df1aa","trusted":true},"cell_type":"code","source":"def regular_encode(texts,tokenizer,maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n    texts,\n    return_attention_masks = False,\n    return_token_type_ids = False,\n    pad_to_max_length = True,\n    max_length = maxlen\n    )\n    \n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"5217cb65-12c9-4507-8099-aea3c7db574d","_cell_guid":"88ab7dd7-6fe0-48f0-9960-83d21c6a1e0a","trusted":true},"cell_type":"code","source":"\n\ndef build_roberta_model(transformer, max_len=512):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    #cls_token = Dense(500, activation=\"relu\")(cls_token)\n    #cls_token = Dropout(0.1)(cls_token)\n    out = Dense(1, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), \n                  loss='binary_crossentropy', \n                  metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"22c3a892-9f15-4fc9-8f53-675fce61fc26","_cell_guid":"b6c8562c-9e05-4e9b-a279-7615d9d5e5d2","trusted":true},"cell_type":"code","source":"#TPU\ntry:\n    \n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU', tpu.master())\nexcept ValueError:\n    tpu=None\n    \nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n    \nprint(\"REPLICAS:\", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b3348738-4dc9-4ff7-8223-d1281f5f61e9","_cell_guid":"499a612a-697c-4eff-b893-9d442df64e42","trusted":true},"cell_type":"markdown","source":"# TPU Configuration"},{"metadata":{"_uuid":"8a326727-399a-4f84-b755-e3677c9036d0","_cell_guid":"d5527bc0-8567-43c5-ad79-146eea75e5be","trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\n\nEPOCHS = 2\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nMAX_LEN = 192\nMODEL = 'jplu/tf-xlm-roberta-large'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"35713757-f5fc-4d03-8c94-8cc49270c193","_cell_guid":"0360f9ab-a058-4f32-ae49-e4babee28468","trusted":true},"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ebda370-4ae2-461b-bb4a-570228819fdd","_cell_guid":"8099beff-f01a-4f1b-b26c-25e2823d169d","trusted":true},"cell_type":"code","source":"train = pd.concat([\n        train1[['comment_text', 'toxic']],\n        train2[['comment_text', 'toxic']].query('toxic == 1'),\n        train2[['comment_text', 'toxic']].query('toxic == 0').sample(n=100000, random_state=0)\n\n])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"566ecc53-a6a8-4b9f-b470-f988cc4c9f74","_cell_guid":"3c22ac97-3459-4b2b-bc55-7dadce302760","trusted":true},"cell_type":"code","source":"%%time\n\nx_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)\n\nx_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen = MAX_LEN)\n\nx_test  = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)\n\ny_train = train.toxic.values\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"73cf6d48-3791-44c8-85a6-bb22829dfdb9","_cell_guid":"50325c85-40b8-423d-8101-604dc629926a","trusted":true},"cell_type":"markdown","source":"**Encoding comments for compatibility and getting targets**"},{"metadata":{"_uuid":"bda9bde4-293b-47d5-b58c-c088fd8985d9","_cell_guid":"eca1a2a8-ad47-4433-a819-44e3cc45f22c","trusted":true},"cell_type":"code","source":"train_dataset = (\n        tf.data.Dataset.from_tensor_slices((x_train,y_train)).repeat().shuffle(2048).batch(BATCH_SIZE).prefetch(AUTO) )\n\n\nvalid_dataset = (\n        tf.data.Dataset.from_tensor_slices((x_valid,y_valid)).batch(BATCH_SIZE).cache().prefetch(AUTO)\n)\n\ntest_dataset = (\n        tf.data.Dataset.from_tensor_slices(x_test).batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"909941c1-bd8e-4a01-b075-040406d2bd40","_cell_guid":"815b465c-73c4-4518-a37b-463c14da0e0a","trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_roberta_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ddb86591-9907-4621-b12b-241e2c995333","_cell_guid":"36cea1c5-04a7-4d18-9774-8513ab40749d","trusted":true},"cell_type":"markdown","source":"# Training"},{"metadata":{"_uuid":"109f1720-6be2-464f-8c04-173681696dcf","_cell_guid":"c48d3fb5-820f-495a-a60f-8438f704b7ff","trusted":true},"cell_type":"code","source":"\n\nN_STEPS = x_train.shape[0] // BATCH_SIZE\n\n\ntrain_history = model.fit(\n                train_dataset, steps_per_epoch=N_STEPS,validation_data=valid_dataset,epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"35690af2-a4b9-4768-973c-9382227bd342","_cell_guid":"0cfd0409-70b1-4e74-81f9-6ad7a3213235","trusted":true},"cell_type":"markdown","source":"# Output"},{"metadata":{"_uuid":"501cf1f3-d874-45db-a8f6-642cc1e2c179","_cell_guid":"b06f6a65-45a7-421e-913b-b24f0df8b440","trusted":true},"cell_type":"code","source":"\nsub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"15b3d61e-c945-45fd-94f5-309f6cd2f6e0","_cell_guid":"951f123d-7511-4900-8444-a66dfd4d619a","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"name":"python","version":"3.6.4","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat":4,"nbformat_minor":4}