{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Data Mining Final Project.\n# Toxic Comment Classification.\n## Authors : Team - 7 \n## Akhil Thakur\n## Spriha Awasthi\n## Ajay Sadananda\n<br>\n<br>\n\n# Project Title : Toxic Comment Classification.\n\nIn this notebook, we will be using machine learning to classify toxic comments. First, we will be conducting EDA on the dataset, which is obtained from kaggle competition ([Link](https://www.kaggle.com/c/jigsaw-multilingual-toxic-comment-classification/data)). Then we use the following models to classify the toxic comments.\n* Simple RNN's\n* LSTM's\n* BI-Directional LSTM's\n* GRU's\n* BERT","metadata":{}},{"cell_type":"markdown","source":"# Loading the libraries\nLets first load all the necessary packages.","metadata":{}},{"cell_type":"code","source":"## Required packages to run the code\n#!pip install tensorflow\n#!pip install pyicu\n#!pip install pycld2\n# !pip install polyglot\n# !pip install textstat\n# !pip install googletrans\n# !pip install plotly==5.4.0","metadata":{"_kg_hide-input":false,"_kg_hide-output":true,"execution":{"iopub.status.busy":"2021-12-09T20:57:01.975672Z","iopub.execute_input":"2021-12-09T20:57:01.975960Z","iopub.status.idle":"2021-12-09T20:57:01.980327Z","shell.execute_reply.started":"2021-12-09T20:57:01.975927Z","shell.execute_reply":"2021-12-09T20:57:01.979489Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom tensorflow.keras.models import Sequential\n#from tensorflow.keras.layers import LSTM, GRU,SimpleRNN\n#from tensorflow.keras.layers import Dense, Activation, Dropout\n#from tensorflow.keras.layers import Embedding\n#from tensorflow.keras.layers import BatchNormalization\n#from tensorflow.keras.utils import np_utils\nfrom tensorflow.keras.utils import to_categorical\n\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\n#from tensorflow.keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D\nfrom tensorflow.keras.preprocessing import sequence, text\nfrom tensorflow.keras.callbacks import EarlyStopping\n\nfrom tensorflow.keras.callbacks import Callback\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\n\nfrom tensorflow.keras.models import Model\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.optimizers import Adam\nfrom tokenizers import BertWordPieceTokenizer\nfrom tensorflow.keras.layers import Dense, Input, Dropout, Embedding\nfrom tensorflow.keras.layers import LSTM, GRU, Conv1D, SpatialDropout1D\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import activations\nfrom tensorflow.keras import constraints\nfrom tensorflow.keras import initializers\nfrom tensorflow.keras import regularizers\n\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.activations import *\nfrom tensorflow.keras.constraints import *\nfrom tensorflow.keras.initializers import *\nfrom tensorflow.keras.regularizers import *\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nimport os\nos.environ['TF_XLA_FLAGS'] = '--tf_xla_enable_xla_devices'\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport gc\nimport re\nimport folium\nimport textstat\nfrom scipy import stats\nfrom colorama import Fore, Back, Style, init\n\nimport math\nimport scipy as sp\n\nimport random\nimport networkx as nx\nfrom pandas import Timestamp\n\nfrom PIL import Image\nfrom IPython.display import SVG\nfrom tensorflow.keras.utils import  model_to_dot\n\nimport requests\nfrom IPython.display import HTML\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\n\ntqdm.pandas()\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\nimport transformers\n\n\nfrom sklearn import metrics\nfrom sklearn.utils import shuffle\nfrom gensim.models import Word2Vec\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_extraction.text import TfidfVectorizer,\\\n                                            CountVectorizer,\\\n                                            HashingVectorizer\n\nfrom nltk.stem.wordnet import WordNetLemmatizer \nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer  \n\nimport nltk\nfrom textblob import TextBlob\n\nfrom nltk.corpus import wordnet\nfrom nltk.corpus import stopwords\nfrom googletrans import Translator\nfrom nltk import WordNetLemmatizer\nfrom polyglot.detect import Detector\nfrom nltk.stem import WordNetLemmatizer\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\n\nstopword=set(STOPWORDS)\n\nlem = WordNetLemmatizer()\ntokenizer=TweetTokenizer()\n\nnp.random.seed(0)","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","execution":{"iopub.status.busy":"2021-12-09T20:57:01.982623Z","iopub.execute_input":"2021-12-09T20:57:01.983001Z","iopub.status.idle":"2021-12-09T20:57:12.386867Z","shell.execute_reply.started":"2021-12-09T20:57:01.982960Z","shell.execute_reply":"2021-12-09T20:57:12.385899Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuring TPU's\n\nWe will be using TPU's to accelerate the training. For this project, TPU's from kaggle are used.","metadata":{}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    #tpu = None\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    \n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T20:57:12.388539Z","iopub.execute_input":"2021-12-09T20:57:12.388785Z","iopub.status.idle":"2021-12-09T20:57:18.215879Z","shell.execute_reply.started":"2021-12-09T20:57:12.388757Z","shell.execute_reply":"2021-12-09T20:57:18.215028Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\nval_data = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest_data = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')","metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","execution":{"iopub.status.busy":"2021-12-09T20:57:18.217466Z","iopub.execute_input":"2021-12-09T20:57:18.217710Z","iopub.status.idle":"2021-12-09T20:57:22.774270Z","shell.execute_reply.started":"2021-12-09T20:57:18.217682Z","shell.execute_reply":"2021-12-09T20:57:22.773466Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Now, Lets have a look at the data.","metadata":{}},{"cell_type":"code","source":"train_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T20:57:22.775916Z","iopub.execute_input":"2021-12-09T20:57:22.776136Z","iopub.status.idle":"2021-12-09T20:57:22.798954Z","shell.execute_reply.started":"2021-12-09T20:57:22.776112Z","shell.execute_reply":"2021-12-09T20:57:22.798337Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T20:57:22.799894Z","iopub.execute_input":"2021-12-09T20:57:22.800565Z","iopub.status.idle":"2021-12-09T20:57:22.810981Z","shell.execute_reply.started":"2021-12-09T20:57:22.800531Z","shell.execute_reply":"2021-12-09T20:57:22.810148Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_data.head()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T20:57:22.812061Z","iopub.execute_input":"2021-12-09T20:57:22.812308Z","iopub.status.idle":"2021-12-09T20:57:22.830161Z","shell.execute_reply.started":"2021-12-09T20:57:22.812272Z","shell.execute_reply":"2021-12-09T20:57:22.829384Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Sentiment and polarity <a id=\"1.4\"></a>\n\nSentiment and polarity are quantities that reflect the emotion and intention behind a sentence. Now, Let's look at the sentiment of the comments using the NLTK (natural language toolkit) library.","metadata":{}},{"cell_type":"code","source":"def polarity(x):\n    if type(x) == str:\n        return SIA.polarity_scores(x)\n    else:\n        return 1000\n\nSIA = SentimentIntensityAnalyzer()\ntrain_data[\"polarity\"] = train_data[\"comment_text\"].progress_apply(polarity)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T20:57:22.831331Z","iopub.execute_input":"2021-12-09T20:57:22.831610Z","iopub.status.idle":"2021-12-09T21:00:46.907436Z","shell.execute_reply.started":"2021-12-09T20:57:22.831582Z","shell.execute_reply":"2021-12-09T21:00:46.906537Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Negative sentiment\n\nNegative sentiment refers to negative or pessimistic emotions. It is a score between 0 and 1; the greater the score, the more negative the subject is.","metadata":{}},{"cell_type":"code","source":"fig = go.Figure(go.Histogram(x=[pols[\"neg\"] for pols in train_data[\"polarity\"] if pols[\"neg\"] != 0], marker=dict(\n            color='seagreen')\n    ))\n\nfig.update_layout(xaxis_title=\"Negativity sentiment\", title_text=\"Negativity sentiment\", template=\"simple_white\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:46.908783Z","iopub.execute_input":"2021-12-09T21:00:46.909099Z","iopub.status.idle":"2021-12-09T21:00:48.600490Z","shell.execute_reply.started":"2021-12-09T21:00:46.909069Z","shell.execute_reply":"2021-12-09T21:00:48.599807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above plot, we can see that negative sentiment has a positive skew, indicating that negativity is usually on the lower side. This suggests that most comments are not toxic or negative.","metadata":{}},{"cell_type":"markdown","source":"### Negativity vs. Toxicity","metadata":{}},{"cell_type":"code","source":"nums_1 = train_data.sample(frac=0.1).query(\"toxic == 1\")\nnums_1 = [pols[\"neg\"] for pols in nums_1[\"polarity\"]]\nnums_2 = train_data.sample(frac=0.1).query(\"toxic == 0\")\nnums_2 = [pols[\"neg\"] for pols in nums_2[\"polarity\"]]\n\n\nfig = ff.create_distplot(hist_data=[nums_1, nums_2],\n                         group_labels=[\"Toxic\", \"Non-toxic\"],\n                         colors=[\"darkorange\", \"dodgerblue\"], show_hist=False)\n\nfig.update_layout(title_text=\"Negativity vs. Toxicity\", xaxis_title=\"Negativity\", template=\"simple_white\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:48.601657Z","iopub.execute_input":"2021-12-09T21:00:48.602001Z","iopub.status.idle":"2021-12-09T21:00:49.386151Z","shell.execute_reply.started":"2021-12-09T21:00:48.601972Z","shell.execute_reply":"2021-12-09T21:00:49.385483Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can clearly see that toxic comments have a significantly greater negative sentiment than toxic comments (on average). The probability density of negativity peaks at around 0 for non-toxic comments, while the negativity for toxic comments are minimum at this point. This suggests that a comment is very likely to be non-toxic if it has a negativity of 0.","metadata":{}},{"cell_type":"markdown","source":"### Positive sentiment\n\nPositive sentiment refers to positive or optimistic emotions. It is a score between 0 and 1; the greater the score, the more positive the subject is.","metadata":{}},{"cell_type":"code","source":"fig = go.Figure(go.Histogram(x=[pols[\"pos\"] for pols in train_data[\"polarity\"] if pols[\"pos\"] != 0], marker=dict(\n            color='indianred')))\n\nfig.update_layout(xaxis_title=\"Positivity sentiment\", title_text=\"Positivity sentiment\", template=\"simple_white\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:49.388689Z","iopub.execute_input":"2021-12-09T21:00:49.389085Z","iopub.status.idle":"2021-12-09T21:00:51.110229Z","shell.execute_reply.started":"2021-12-09T21:00:49.389041Z","shell.execute_reply":"2021-12-09T21:00:51.109416Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above plot, we can see that positive sentiment has a positive skew, indicating that positivity is usually on the lower side. This suggests that most comments do not express positivity explicitly","metadata":{}},{"cell_type":"markdown","source":"### Positivity vs. Toxicity","metadata":{}},{"cell_type":"code","source":"nums_1 = train_data.sample(frac=0.1).query(\"toxic == 1\")\nnums_1 = [pols[\"pos\"] for pols in nums_1[\"polarity\"]]\nnums_2 = train_data.sample(frac=0.1).query(\"toxic == 0\")\nnums_2 = [pols[\"pos\"] for pols in nums_2[\"polarity\"]]\n\n\nfig = ff.create_distplot(hist_data=[nums_1, nums_2],\n                         group_labels=[\"Toxic\", \"Non-toxic\"],\n                         colors=[\"darkorange\", \"dodgerblue\"], show_hist=False)\n\nfig.update_layout(title_text=\"Positivity vs. Toxicity\", xaxis_title=\"Positivity\", template=\"simple_white\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:51.111599Z","iopub.execute_input":"2021-12-09T21:00:51.111827Z","iopub.status.idle":"2021-12-09T21:00:51.846739Z","shell.execute_reply.started":"2021-12-09T21:00:51.111800Z","shell.execute_reply":"2021-12-09T21:00:51.846098Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Here we have plotted the distribution of positivity for toxic and non-toxic comments above. We can see that both the distributions are very similar, indicating that positivity is not an accurate indicator of toxicity in comments.","metadata":{}},{"cell_type":"markdown","source":"### Neutrality sentiment\n\nNeutrality sentiment refers to the level of bias or opinion in the text. It is a score between 0 and 1; the greater the score, the more neutral/unbiased the subject is.","metadata":{}},{"cell_type":"code","source":"fig = go.Figure(go.Histogram(x=[pols[\"neu\"] for pols in train_data[\"polarity\"] if pols[\"neu\"] != 1], marker=dict(\n            color='dodgerblue')\n    ))\n\nfig.update_layout(xaxis_title=\"Neutrality sentiment\", title_text=\"Neutrality sentiment\", template=\"simple_white\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:51.847976Z","iopub.execute_input":"2021-12-09T21:00:51.848354Z","iopub.status.idle":"2021-12-09T21:00:53.870921Z","shell.execute_reply.started":"2021-12-09T21:00:51.848309Z","shell.execute_reply":"2021-12-09T21:00:53.870006Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the above plot, we can see that the neutrality sentiment distribution has a negative skew, which is in constrast to the negativity and positivity sentiment distributions. This indicates that the comments tend to be very neutral and unbiased in general. ","metadata":{}},{"cell_type":"markdown","source":"### Neutrality vs. Toxicity","metadata":{}},{"cell_type":"code","source":"nums_1 = train_data.sample(frac=0.1).query(\"toxic == 1\")\nnums_1 = [pols[\"neu\"] for pols in nums_1[\"polarity\"]]\nnums_2 = train_data.sample(frac=0.1).query(\"toxic == 0\")\nnums_2 = [pols[\"neu\"] for pols in nums_2[\"polarity\"]]\n\nfig = ff.create_distplot(hist_data=[nums_1, nums_2],\n                         group_labels=[\"Toxic\", \"Non-toxic\"],\n                         colors=[\"darkorange\", \"dodgerblue\"], show_hist=False)\n\nfig.update_layout(title_text=\"Neutrality vs. Toxicity\", xaxis_title=\"Neutrality\", template=\"simple_white\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:53.872101Z","iopub.execute_input":"2021-12-09T21:00:53.872342Z","iopub.status.idle":"2021-12-09T21:00:54.630438Z","shell.execute_reply.started":"2021-12-09T21:00:53.872314Z","shell.execute_reply":"2021-12-09T21:00:54.629558Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that non-toxic comments tend to have a higher neutrality value than toxic comments on average. The probability density of the non-toxic distribution experiences a sudden jump at 1, and the probability density of the toxic distribution is significantly lower at the same point. This suggests that a comment with neutrality close to 1 is more likely to be non-toxic than toxic.","metadata":{}},{"cell_type":"markdown","source":"### Compound sentiment\n\nCompoundness sentiment refers to the total level of sentiment in the sentence. It is a score between -1 and 1; the greater the score, the more emotional the subject is.","metadata":{}},{"cell_type":"code","source":"fig = go.Figure(go.Histogram(x=[pols[\"compound\"] for pols in train_data[\"polarity\"] if pols[\"compound\"] != 0], marker=dict(\n            color='orchid')\n    ))\n\nfig.update_layout(xaxis_title=\"Compound sentiment\", title_text=\"Compound sentiment\", template=\"simple_white\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:54.631840Z","iopub.execute_input":"2021-12-09T21:00:54.632073Z","iopub.status.idle":"2021-12-09T21:00:56.648411Z","shell.execute_reply.started":"2021-12-09T21:00:54.632045Z","shell.execute_reply":"2021-12-09T21:00:56.647534Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the distribution above, we can see that compound sentiment is evenly distributed across the specturm (from -1 to 1) with very high variance and random peaks throughout the range.","metadata":{}},{"cell_type":"markdown","source":"### Compound sentiment vs. Toxicity","metadata":{}},{"cell_type":"code","source":"nums_1 = train_data.sample(frac=0.1).query(\"toxic == 1\")\nnums_1 = [pols[\"compound\"] for pols in nums_1[\"polarity\"]]\nnums_2 = train_data.sample(frac=0.1).query(\"toxic == 0\")\nnums_2 = [pols[\"compound\"] for pols in nums_2[\"polarity\"]]\n\n\nfig = ff.create_distplot(hist_data=[nums_1, nums_2],\n                         group_labels=[\"Toxic\", \"Non-toxic\"],\n                         colors=[\"darkorange\", \"dodgerblue\"], show_hist=False)\n\nfig.update_layout(title_text=\"Compound vs. Toxicity\", xaxis_title=\"Compound\", template=\"simple_white\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:56.649859Z","iopub.execute_input":"2021-12-09T21:00:56.650141Z","iopub.status.idle":"2021-12-09T21:00:57.372624Z","shell.execute_reply.started":"2021-12-09T21:00:56.650112Z","shell.execute_reply":"2021-12-09T21:00:57.371787Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that compound sentiment tends to be higher for non-toxic comments as compared to toxic comments. The non-toxic distribution has a negative skew, while the toxic distribution has a positive skew. This indicates that non-toxic comments tend to have a higher compound sentiment than toxic comments on average.","metadata":{}},{"cell_type":"markdown","source":"## Targets <a id=\"1.6\"></a>\n\nTargets are the outputs of our classification. Here we have various different toxicity classes such as severe_toxic, obscene, threat, insult, identity_hate. Now, Let's visualize the targets in the dataset.","metadata":{}},{"cell_type":"code","source":"fig = go.Figure(data=[\n    go.Pie(labels=train_data.columns[2:7],\n           values=train_data.iloc[:, 2:7].sum().values, marker=dict(colors=px.colors.qualitative.Plotly))\n])\nfig.update_traces(textposition='outside', textfont=dict(color=\"black\"))\nfig.update_layout(title_text=\"Pie chart of labels\")\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:57.373814Z","iopub.execute_input":"2021-12-09T21:00:57.374461Z","iopub.status.idle":"2021-12-09T21:00:57.401790Z","shell.execute_reply.started":"2021-12-09T21:00:57.374428Z","shell.execute_reply":"2021-12-09T21:00:57.401202Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"From the pie chart above, we can see that the most common target is toxic, and the other targets, such as insult and threat are relatively uncommon.","metadata":{}},{"cell_type":"markdown","source":"# Toxic comment classification as a binary classification problem.\n\n* We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models.","metadata":{}},{"cell_type":"code","source":"train_data.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:57.402902Z","iopub.execute_input":"2021-12-09T21:00:57.403266Z","iopub.status.idle":"2021-12-09T21:00:57.430495Z","shell.execute_reply.started":"2021-12-09T21:00:57.403223Z","shell.execute_reply":"2021-12-09T21:00:57.429613Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_data = train_data.loc[:12000,:]\ntrain_data.shape","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:57.431758Z","iopub.execute_input":"2021-12-09T21:00:57.432009Z","iopub.status.idle":"2021-12-09T21:00:57.438652Z","shell.execute_reply.started":"2021-12-09T21:00:57.431974Z","shell.execute_reply":"2021-12-09T21:00:57.437873Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will check the maximum number of words that can be present in a comment , this will help us in padding later","metadata":{}},{"cell_type":"code","source":"train_data['comment_text'].apply(lambda x:len(str(x).split())).max()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:57.439834Z","iopub.execute_input":"2021-12-09T21:00:57.440655Z","iopub.status.idle":"2021-12-09T21:00:57.518984Z","shell.execute_reply.started":"2021-12-09T21:00:57.440613Z","shell.execute_reply":"2021-12-09T21:00:57.518124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Writing a function for getting auc score for validation","metadata":{}},{"cell_type":"code","source":"def roc_auc(predictions,target):\n    '''\n    This methods returns the AUC Score when given the Predictions\n    and Labels\n    '''\n    \n    fpr, tpr, thresholds = metrics.roc_curve(target, predictions)\n    roc_auc = metrics.auc(fpr, tpr)\n    return roc_auc","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:57.520313Z","iopub.execute_input":"2021-12-09T21:00:57.521067Z","iopub.status.idle":"2021-12-09T21:00:57.526994Z","shell.execute_reply.started":"2021-12-09T21:00:57.521019Z","shell.execute_reply":"2021-12-09T21:00:57.526325Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"### Data Preparation","metadata":{}},{"cell_type":"code","source":"xtrain, xvalid, ytrain, yvalid = train_test_split(train_data.comment_text.values, train_data.toxic.values, \n                                                  stratify=train_data.toxic.values, \n                                                  random_state=42, \n                                                  test_size=0.2, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:57.528090Z","iopub.execute_input":"2021-12-09T21:00:57.528985Z","iopub.status.idle":"2021-12-09T21:00:57.555554Z","shell.execute_reply.started":"2021-12-09T21:00:57.528946Z","shell.execute_reply":"2021-12-09T21:00:57.554688Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Simple RNN\n\nRecurrent Neural Network(RNN) are a type of Neural Network where the output from previous step are fed as input to the current step. In traditional neural networks, all the inputs and outputs are independent of each other, but in cases like when it is required to predict the next word of a sentence, the previous words are required and hence there is a need to remember the previous words. Thus RNN came into existence, which solved this issue with the help of a Hidden Layer.","metadata":{}},{"cell_type":"code","source":"# using keras tokenizer here\ntoken = text.Tokenizer(num_words=None)\nmax_len = 1500\n\ntoken.fit_on_texts(list(xtrain) + list(xvalid))\nxtrain_seq = token.texts_to_sequences(xtrain)\nxvalid_seq = token.texts_to_sequences(xvalid)\n\n#zero pad the sequences\nxtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)\nxvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)\n\nword_index = token.word_index","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:57.557172Z","iopub.execute_input":"2021-12-09T21:00:57.557505Z","iopub.status.idle":"2021-12-09T21:00:59.498547Z","shell.execute_reply.started":"2021-12-09T21:00:57.557464Z","shell.execute_reply":"2021-12-09T21:00:59.497579Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith strategy.scope():\n    # A simpleRNN without any pretrained embeddings and one dense layer\n    model = Sequential()\n    model.add(Embedding(len(word_index) + 1,\n                     300,\n                     input_length=max_len))\n    model.add(SimpleRNN(100))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'], steps_per_execution=32)\n    \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:00:59.499902Z","iopub.execute_input":"2021-12-09T21:00:59.500648Z","iopub.status.idle":"2021-12-09T21:01:00.294536Z","shell.execute_reply.started":"2021-12-09T21:00:59.500604Z","shell.execute_reply":"2021-12-09T21:01:00.293644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#early_stop = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy',patience=2, restore_best_weights=True)\nmodel.fit(xtrain_pad, ytrain,epochs=5, batch_size=16*strategy.num_replicas_in_sync) #Multiplying by Strategy to run on TPU's","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:01:00.297519Z","iopub.execute_input":"2021-12-09T21:01:00.297757Z","iopub.status.idle":"2021-12-09T21:01:16.753705Z","shell.execute_reply.started":"2021-12-09T21:01:00.297730Z","shell.execute_reply":"2021-12-09T21:01:16.752803Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = model.predict(xvalid_pad)\nprint(\"Auc: %.2f%%\" % (roc_auc(scores,yvalid)))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:01:16.755266Z","iopub.execute_input":"2021-12-09T21:01:16.755695Z","iopub.status.idle":"2021-12-09T21:01:20.481751Z","shell.execute_reply.started":"2021-12-09T21:01:16.755653Z","shell.execute_reply":"2021-12-09T21:01:20.481091Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores_model = []\nscores_model.append({'Model': 'SimpleRNN','AUC_Score': roc_auc(scores,yvalid)})","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:01:20.483019Z","iopub.execute_input":"2021-12-09T21:01:20.483509Z","iopub.status.idle":"2021-12-09T21:01:20.490113Z","shell.execute_reply.started":"2021-12-09T21:01:20.483464Z","shell.execute_reply":"2021-12-09T21:01:20.489330Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary\n* Tokenization<br><br>\n In an RNN we input a sentence word by word. We represent every word as one hot vectors of dimensions : Numbers of words in Vocab +1. <br>\n  The keras Tokenizer takes all the unique words in the corpus,forms a dictionary with words as keys and their number of occurrences as values,it then sorts the dictionary in descending order of counts. It then assigns the first value 1 , second value 2 and so on. So let's suppose word 'the' occurred the most in the corpus then it will assign index 1 and vector representing 'the' would be a one-hot vector with value 1 at position 1 and rest zeros.<br>","metadata":{}},{"cell_type":"markdown","source":"* Comments on the model<br><br>\nWe can see our model achieves an accuracy of almost 1 which is just insane , we are clearly overfitting I know , but this was the simplest model of all ,we can tune a lot of hyperparameters like RNN units, we can do batch normalization , dropouts etc to get better result. The point is we got an AUC score of 0.82 without much efforts.","metadata":{}},{"cell_type":"markdown","source":"# Word Embeddings\n<br>\nOne of the approach to getting word Embeddings is using pretained GLoVe. In this Notebook, we'll be using the GloVe vectors. You can download the GloVe vectors from here http://www-nlp.stanford.edu/data/glove.840B.300d.zip or you can search for GloVe in datasets on Kaggle and add the file","metadata":{}},{"cell_type":"code","source":"# load the GloVe vectors in a dictionary:\n\nembeddings_index = {}\nf = open('/kaggle/input/glove840b300dtxt/glove.840B.300d.txt','r',encoding='utf-8')\nfor line in tqdm(f):\n    values = line.split(' ')\n    word = values[0]\n    coefs = np.asarray([float(val) for val in values[1:]])\n    embeddings_index[word] = coefs\nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:01:20.491096Z","iopub.execute_input":"2021-12-09T21:01:20.491307Z","iopub.status.idle":"2021-12-09T21:06:39.141406Z","shell.execute_reply.started":"2021-12-09T21:01:20.491283Z","shell.execute_reply":"2021-12-09T21:06:39.140497Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# LSTM's\n\nSimple RNN's were certainly better than classical ML algorithms and gave state of the art results, but it failed to capture long term dependencies that is present in sentences . So in 1998-99 LSTM's were introduced to counter to these drawbacks. We have already tokenized and paded our text for input to LSTM's","metadata":{}},{"cell_type":"code","source":"# create an embedding matrix for the words we have in the dataset\nembedding_matrix = np.zeros((len(word_index) + 1, 300))\nfor word, i in tqdm(word_index.items()):\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:06:39.147657Z","iopub.execute_input":"2021-12-09T21:06:39.147997Z","iopub.status.idle":"2021-12-09T21:06:39.336479Z","shell.execute_reply.started":"2021-12-09T21:06:39.147965Z","shell.execute_reply":"2021-12-09T21:06:39.335541Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith strategy.scope():\n    \n    # A simple LSTM with glove embeddings and one dense layer\n    model = Sequential()\n    model.add(Embedding(len(word_index) + 1,\n                     300,\n                     weights=[embedding_matrix],\n                     input_length=max_len,\n                     trainable=False))\n\n    model.add(LSTM(100, dropout=0.3, recurrent_dropout=0.3))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])\n    \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:06:39.338080Z","iopub.execute_input":"2021-12-09T21:06:39.338300Z","iopub.status.idle":"2021-12-09T21:06:41.367571Z","shell.execute_reply.started":"2021-12-09T21:06:39.338275Z","shell.execute_reply":"2021-12-09T21:06:41.366580Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(xtrain_pad, ytrain, epochs=5, batch_size=16*strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:06:41.369161Z","iopub.execute_input":"2021-12-09T21:06:41.369732Z","iopub.status.idle":"2021-12-09T21:07:16.304326Z","shell.execute_reply.started":"2021-12-09T21:06:41.369687Z","shell.execute_reply":"2021-12-09T21:07:16.303331Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = model.predict(xvalid_pad)\nprint(\"Auc: %.2f%%\" % (roc_auc(scores,yvalid)))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:07:16.305592Z","iopub.execute_input":"2021-12-09T21:07:16.305838Z","iopub.status.idle":"2021-12-09T21:07:19.587699Z","shell.execute_reply.started":"2021-12-09T21:07:16.305811Z","shell.execute_reply":"2021-12-09T21:07:19.586775Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores_model.append({'Model': 'LSTM','AUC_Score': roc_auc(scores,yvalid)})","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:07:19.589042Z","iopub.execute_input":"2021-12-09T21:07:19.589561Z","iopub.status.idle":"2021-12-09T21:07:19.596762Z","shell.execute_reply.started":"2021-12-09T21:07:19.589521Z","shell.execute_reply":"2021-12-09T21:07:19.595917Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary\n\nAs a first step we calculate embedding matrix for our vocabulary from the pretrained GLoVe vectors . Then while building the embedding layer we pass Embedding Matrix as weights to the layer instead of training it over Vocabulary and thus we pass trainable = False.\nRest of the model is same as before except we have replaced the SimpleRNN By LSTM Units\n\n* Comments on the Model\n\nWe now see that the model is not overfitting and achieves an auc score of 0.96 which is quite fair, also we close in on the gap between accuracy and auc .\nWe see that in this case we used dropout and prevented overfitting the data","metadata":{}},{"cell_type":"markdown","source":"# GRU's\n\nIntroduced by Cho, et al. in 2014, GRU (Gated Recurrent Unit) aims to solve the vanishing gradient problem which comes with a standard recurrent neural network. GRU's are a variation on the LSTM because both are designed similarly and, in some cases, produce equally excellent results . GRU's were designed to be simpler and faster than LSTM's and in most cases produce equally good results and thus there is no clear winner.\n","metadata":{}},{"cell_type":"code","source":"%%time\nwith strategy.scope():\n    # GRU with glove embeddings and two dense layers\n     model = Sequential()\n     model.add(Embedding(len(word_index) + 1,\n                     300,\n                     weights=[embedding_matrix],\n                     input_length=max_len,\n                     trainable=False))\n     model.add(SpatialDropout1D(0.3))\n     model.add(GRU(300))\n     model.add(Dense(1, activation='sigmoid'))\n\n     model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])   \n    \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:07:19.598478Z","iopub.execute_input":"2021-12-09T21:07:19.598825Z","iopub.status.idle":"2021-12-09T21:07:22.615861Z","shell.execute_reply.started":"2021-12-09T21:07:19.598748Z","shell.execute_reply":"2021-12-09T21:07:22.615130Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(xtrain_pad, ytrain, epochs=5, batch_size=16*strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:07:22.616824Z","iopub.execute_input":"2021-12-09T21:07:22.617185Z","iopub.status.idle":"2021-12-09T21:07:56.496305Z","shell.execute_reply.started":"2021-12-09T21:07:22.617156Z","shell.execute_reply":"2021-12-09T21:07:56.495430Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = model.predict(xvalid_pad)\nprint(\"Auc: %.2f%%\" % (roc_auc(scores,yvalid)))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:07:56.497708Z","iopub.execute_input":"2021-12-09T21:07:56.498037Z","iopub.status.idle":"2021-12-09T21:07:59.388738Z","shell.execute_reply.started":"2021-12-09T21:07:56.497997Z","shell.execute_reply":"2021-12-09T21:07:59.387757Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores_model.append({'Model': 'GRU','AUC_Score': roc_auc(scores,yvalid)})","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:07:59.390102Z","iopub.execute_input":"2021-12-09T21:07:59.390430Z","iopub.status.idle":"2021-12-09T21:07:59.397151Z","shell.execute_reply.started":"2021-12-09T21:07:59.390388Z","shell.execute_reply":"2021-12-09T21:07:59.396242Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores_model","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:07:59.398434Z","iopub.execute_input":"2021-12-09T21:07:59.398756Z","iopub.status.idle":"2021-12-09T21:07:59.409550Z","shell.execute_reply.started":"2021-12-09T21:07:59.398715Z","shell.execute_reply":"2021-12-09T21:07:59.408846Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Summary\n\n* Comments on the Model\n\nWe now see that the model achieves an higher auc score of 0.978 compared to the previous two models. Using GRU, we can see that with almost same accuracy we achieved a higher auc when compared with LSTM.","metadata":{}},{"cell_type":"markdown","source":"# Bi-Directional LSTM\n<br>\nA Bidirectional LSTM, or biLSTM, is a sequence processing model that consists of two LSTMs: one taking the input in a forward direction, and the other in a backwards direction. BiLSTMs effectively increase the amount of information available to the network, improving the context available to the algorithm ","metadata":{}},{"cell_type":"code","source":"%%time\nwith strategy.scope():\n    # A simple bidirectional LSTM with glove embeddings and one dense layer\n    model = Sequential()\n    model.add(Embedding(len(word_index) + 1,\n                     300,\n                     weights=[embedding_matrix],\n                     input_length=max_len,\n                     trainable=False))\n    model.add(Bidirectional(LSTM(300, dropout=0.3, recurrent_dropout=0.3)))\n\n    model.add(Dense(1,activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])\n    \n    \nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:07:59.411725Z","iopub.execute_input":"2021-12-09T21:07:59.412089Z","iopub.status.idle":"2021-12-09T21:08:03.555789Z","shell.execute_reply.started":"2021-12-09T21:07:59.412060Z","shell.execute_reply":"2021-12-09T21:08:03.554944Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"model.fit(xtrain_pad, ytrain, epochs=5, batch_size=16*strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:08:03.557030Z","iopub.execute_input":"2021-12-09T21:08:03.557252Z","iopub.status.idle":"2021-12-09T21:10:20.812841Z","shell.execute_reply.started":"2021-12-09T21:08:03.557220Z","shell.execute_reply":"2021-12-09T21:10:20.811900Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = model.predict(xvalid_pad)\nprint(\"Auc: %.2f%%\" % (roc_auc(scores,yvalid)))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:20.814509Z","iopub.execute_input":"2021-12-09T21:10:20.814989Z","iopub.status.idle":"2021-12-09T21:10:27.337651Z","shell.execute_reply.started":"2021-12-09T21:10:20.814943Z","shell.execute_reply":"2021-12-09T21:10:27.336755Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores_model.append({'Model': 'Bi-directional LSTM','AUC_Score': roc_auc(scores,yvalid)})","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.339206Z","iopub.execute_input":"2021-12-09T21:10:27.339541Z","iopub.status.idle":"2021-12-09T21:10:27.346077Z","shell.execute_reply.started":"2021-12-09T21:10:27.339509Z","shell.execute_reply":"2021-12-09T21:10:27.344834Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We can see that GRU has achived a higher AUC scores among the models we have used. It is important to note that this results might not translate to multi-class classification models.","metadata":{}},{"cell_type":"markdown","source":"## Summary\n\nHere, we have only added bidirectional nature to the LSTM cells we used before and is self explanatory. We have achieve similar accuracy and auc score as before.","metadata":{}},{"cell_type":"markdown","source":"# BERT","metadata":{}},{"cell_type":"code","source":"train1 = pd.read_csv(\"/jigsaw-data/jigsaw-toxic-comment-train.csv\")\nvalid = pd.read_csv('/jigsaw-data/validation.csv')\ntest = pd.read_csv('/jigsaw-data/test.csv')\nsub = pd.read_csv('/jigsaw-data/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.347869Z","iopub.execute_input":"2021-12-09T21:10:27.348199Z","iopub.status.idle":"2021-12-09T21:10:27.358542Z","shell.execute_reply.started":"2021-12-09T21:10:27.348160Z","shell.execute_reply":"2021-12-09T21:10:27.357644Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n    \"\"\"\n    Encoder for encoding the text into sequence of integers for BERT Input\n    \"\"\"\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(pad_id=3, pad_token=\"[PAD]\")\n    all_ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.359931Z","iopub.execute_input":"2021-12-09T21:10:27.360614Z","iopub.status.idle":"2021-12-09T21:10:27.370653Z","shell.execute_reply.started":"2021-12-09T21:10:27.360572Z","shell.execute_reply":"2021-12-09T21:10:27.369726Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"#IMP DATA FOR CONFIG\n\nAUTO = tf.data.experimental.AUTOTUNE\n\n\n# Configuration\nEPOCHS = 5\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nMAX_LEN = 192","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.372262Z","iopub.execute_input":"2021-12-09T21:10:27.372921Z","iopub.status.idle":"2021-12-09T21:10:27.385957Z","shell.execute_reply.started":"2021-12-09T21:10:27.372876Z","shell.execute_reply":"2021-12-09T21:10:27.385122Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# First load the real tokenizer\ntokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\n# tokenizer = AutoTokenizer.from_pretrained('distilbert-base-multilingual-cased')\n# Save the loaded tokenizer locally\ntokenizer.save_pretrained('.')\n# Reload it with the huggingface tokenizers library\nfast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\nfast_tokenizer","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.387085Z","iopub.execute_input":"2021-12-09T21:10:27.387428Z","iopub.status.idle":"2021-12-09T21:10:27.396604Z","shell.execute_reply.started":"2021-12-09T21:10:27.387391Z","shell.execute_reply":"2021-12-09T21:10:27.395772Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)\n\ny_train = train1.toxic.values\ny_valid = valid.toxic.values","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.397674Z","iopub.execute_input":"2021-12-09T21:10:27.397987Z","iopub.status.idle":"2021-12-09T21:10:27.407951Z","shell.execute_reply.started":"2021-12-09T21:10:27.397948Z","shell.execute_reply":"2021-12-09T21:10:27.407037Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.409305Z","iopub.execute_input":"2021-12-09T21:10:27.410226Z","iopub.status.idle":"2021-12-09T21:10:27.419166Z","shell.execute_reply.started":"2021-12-09T21:10:27.410182Z","shell.execute_reply":"2021-12-09T21:10:27.418533Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    \"\"\"\n    function for training the BERT model\n    \"\"\"\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    out = Dense(1, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.420331Z","iopub.execute_input":"2021-12-09T21:10:27.420623Z","iopub.status.idle":"2021-12-09T21:10:27.430387Z","shell.execute_reply.started":"2021-12-09T21:10:27.420593Z","shell.execute_reply":"2021-12-09T21:10:27.429429Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-multilingual-cased')\n    )\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.431939Z","iopub.execute_input":"2021-12-09T21:10:27.432446Z","iopub.status.idle":"2021-12-09T21:10:27.446501Z","shell.execute_reply.started":"2021-12-09T21:10:27.432404Z","shell.execute_reply":"2021-12-09T21:10:27.445481Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_steps = x_train.shape[0] // BATCH_SIZE\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS\n)","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.448753Z","iopub.execute_input":"2021-12-09T21:10:27.449004Z","iopub.status.idle":"2021-12-09T21:10:27.458896Z","shell.execute_reply.started":"2021-12-09T21:10:27.448976Z","shell.execute_reply":"2021-12-09T21:10:27.458092Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores = model.predict(x_valid, verbose=1)\n# scores\n# print(\"Auc: %.2f%%\" % (roc_auc(scores, y_valid)))","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.459952Z","iopub.execute_input":"2021-12-09T21:10:27.460528Z","iopub.status.idle":"2021-12-09T21:10:27.470010Z","shell.execute_reply.started":"2021-12-09T21:10:27.460483Z","shell.execute_reply":"2021-12-09T21:10:27.469328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"n_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=EPOCHS*2\n)\nscores = model.predict(x_valid, verbose=1)\n","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:10:27.471042Z","iopub.execute_input":"2021-12-09T21:10:27.471685Z","iopub.status.idle":"2021-12-09T21:10:27.481078Z","shell.execute_reply.started":"2021-12-09T21:10:27.471647Z","shell.execute_reply":"2021-12-09T21:10:27.480280Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"scores_model = [{'Model': 'SimpleRNN', 'AUC_Score': 0.8262},\n {'Model': 'LSTM', 'AUC_Score': 0.9616},\n {'Model': 'GRU', 'AUC_Score': 0.9736},\n {'Model': 'Bi-directional LSTM', 'AUC_Score': 0.9662},\n {'Model': 'BERT', 'AUC_Score': 0.9779}]\nscores_model","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:11:41.771822Z","iopub.execute_input":"2021-12-09T21:11:41.772168Z","iopub.status.idle":"2021-12-09T21:11:41.780621Z","shell.execute_reply.started":"2021-12-09T21:11:41.772138Z","shell.execute_reply":"2021-12-09T21:11:41.779771Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"# Visualization of Results obtained from various Deep learning models\nresults = pd.DataFrame(scores_model).sort_values(by='AUC_Score',ascending=False)\nresults.style.background_gradient(cmap='Blues')\nfig = go.Figure(go.Funnelarea(\n    text =results.Model,\n    values = results.AUC_Score,\n    title = {\"position\": \"top center\", \"text\": \"Funnel-Chart of AOC Score Distribution\"}\n    ))\nfig.show()","metadata":{"execution":{"iopub.status.busy":"2021-12-09T21:11:50.830402Z","iopub.execute_input":"2021-12-09T21:11:50.831144Z","iopub.status.idle":"2021-12-09T21:11:50.847309Z","shell.execute_reply.started":"2021-12-09T21:11:50.831106Z","shell.execute_reply":"2021-12-09T21:11:50.846731Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"## Code Explanation\n\nHere, we can see that using BERT architecture yielded better results. We have achieved an AUC score of 0.9779, which is the best amongst all the previous models.","metadata":{}},{"cell_type":"markdown","source":"# Summary of the project.\n<br>\n<br>\n\nWe used the jigsaw multilinugual toxic comments data set from Kaggle and performed various EDA's on the data set. After exploring the data, we calculated the sentiment scores for the comments and check how the sentiment scores correlate with the toxicity of the comments. To make the process of training easier, we have reduced our multi-class classification model to a binary classification problem, where a comment could be either toxic or non-toxic. We then used few of the most common ML models namely, RNN, LSTM, GRU, Bidirectional LSTM, BERT to classify the comments. We have observed that the RNN has an overfitting problem and the other three models perform fairly similar.\n","metadata":{}}]}