{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Implementation of XLM-Roberta and DistilBert Transformer models for classification of Multilinguistic Toxic Comments"},{"metadata":{},"cell_type":"markdown","source":"# Model-1 XLM-Roberta Model"},{"metadata":{},"cell_type":"markdown","source":"**Importing required Libraries**"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install --upgrade pip\n!pip install -q textstat","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport gc\nimport os\nimport time\nimport math\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom datetime import date\nfrom transformers import *\nfrom sklearn.metrics import *\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nimport torch.nn.functional as F\n\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport gc\nimport re\nimport folium\nimport textstat\nfrom scipy import stats\nfrom colorama import Fore, Back, Style, init\n\nimport math\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\n\nimport random\nimport networkx as nx\nfrom pandas import Timestamp\n\nfrom PIL import Image\nfrom IPython.display import SVG\nfrom keras.utils import model_to_dot\n\nimport requests\nfrom IPython.display import HTML\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\n\ntqdm.pandas()\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\nimport transformers\nimport tensorflow as tf\n\nfrom tensorflow.keras.callbacks import Callback\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\n\nfrom tensorflow.keras.models import Model\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.optimizers import Adam\nfrom tokenizers import BertWordPieceTokenizer\nfrom tensorflow.keras.layers import Dense, Input, Dropout, Embedding\nfrom tensorflow.keras.layers import LSTM, GRU, Conv1D, SpatialDropout1D\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import activations\nfrom tensorflow.keras import constraints\nfrom tensorflow.keras import initializers\nfrom tensorflow.keras import regularizers\n\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.activations import *\nfrom tensorflow.keras.constraints import *\nfrom tensorflow.keras.initializers import *\nfrom tensorflow.keras.regularizers import *\n\nfrom sklearn import metrics\nfrom sklearn.utils import shuffle\nfrom gensim.models import Word2Vec\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_extraction.text import TfidfVectorizer,\\\n                                            CountVectorizer,\\\n                                            HashingVectorizer\n\nfrom nltk.stem.wordnet import WordNetLemmatizer \nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer  \n\nimport nltk\nfrom textblob import TextBlob\n\nfrom nltk.corpus import wordnet\nfrom nltk.corpus import stopwords\nfrom nltk import WordNetLemmatizer\nfrom nltk.stem import WordNetLemmatizer\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\n\nstopword=set(STOPWORDS)\n\nlem = WordNetLemmatizer()\ntokenizer=TweetTokenizer()\n\nnp.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Setup TPU configuration**"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\nprint(strategy.num_replicas_in_sync)\n\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Importing required datasets from .CSV files**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\n\n\ntrain2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\ntrain2.toxic = train2.toxic.round().astype(int)\n\ntrain3 = pd.read_csv('../input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')\n\n\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\n\n\ntoxic = len(train2[['comment_text', 'toxic']].query('toxic==1'))\n# Combine train1 with a subset of train2\ntrain_cat = pd.concat([\n    train1[['comment_text', 'toxic']],\n    train2[['comment_text', 'toxic']].query('toxic==1'),\n    \n    train3[['comment_text', 'toxic']].query('toxic==0'),\n    train3[['comment_text', 'toxic']].query('toxic==1'),   \n  \n]).sample(n=500000).reset_index(drop=True) #restricting data to 500,000 records due to memory issue\n\ntest_data = test\ntrain_data = train_cat\n\nmaxlen = 192","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_data))\ntrain_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing"},{"metadata":{},"cell_type":"markdown","source":"**Clean the text (remove usernames and links)**"},{"metadata":{"trusted":true},"cell_type":"code","source":"val = valid\ntrain = train_data\n\ndef clean(text):\n    text = text.fillna(\"fillna\").str.lower()\n    text = text.map(lambda x: re.sub('\\\\n',' ',str(x)))\n    text = text.map(lambda x: re.sub(\"\\[\\[User.*\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n    return text\n\nval[\"comment_text\"] = clean(val[\"comment_text\"])\ntest_data[\"content\"] = clean(test_data[\"content\"])\ntrain[\"comment_text\"] = clean(train[\"comment_text\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# More Text Cleaning"},{"metadata":{},"cell_type":"markdown","source":"Applying text cleaning techniques like clean_text,replace_typical_misspell,handle_contractions,fix_quote on train,test and validation set"},{"metadata":{"trusted":true},"cell_type":"code","source":"puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£',\n '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\\xa0', '\\t',\n '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\\u3000', '\\u202f',\n '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',\n '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√']\n\nmispell_dict = {\"aren't\" : \"are not\",\n\"can't\" : \"cannot\",\n\"couldn't\" : \"could not\",\n\"couldnt\" : \"could not\",\n\"didn't\" : \"did not\",\n\"doesn't\" : \"does not\",\n\"doesnt\" : \"does not\",\n\"don't\" : \"do not\",\n\"hadn't\" : \"had not\",\n\"hasn't\" : \"has not\",\n\"haven't\" : \"have not\",\n\"havent\" : \"have not\",\n\"he'd\" : \"he would\",\n\"he'll\" : \"he will\",\n\"he's\" : \"he is\",\n\"i'd\" : \"I would\",\n\"i'd\" : \"I had\",\n\"i'll\" : \"I will\",\n\"i'm\" : \"I am\",\n\"isn't\" : \"is not\",\n\"it's\" : \"it is\",\n\"it'll\":\"it will\",\n\"i've\" : \"I have\",\n\"let's\" : \"let us\",\n\"mightn't\" : \"might not\",\n\"mustn't\" : \"must not\",\n\"shan't\" : \"shall not\",\n\"she'd\" : \"she would\",\n\"she'll\" : \"she will\",\n\"she's\" : \"she is\",\n\"shouldn't\" : \"should not\",\n\"shouldnt\" : \"should not\",\n\"that's\" : \"that is\",\n\"thats\" : \"that is\",\n\"there's\" : \"there is\",\n\"theres\" : \"there is\",\n\"they'd\" : \"they would\",\n\"they'll\" : \"they will\",\n\"they're\" : \"they are\",\n\"theyre\":  \"they are\",\n\"they've\" : \"they have\",\n\"we'd\" : \"we would\",\n\"we're\" : \"we are\",\n\"weren't\" : \"were not\",\n\"we've\" : \"we have\",\n\"what'll\" : \"what will\",\n\"what're\" : \"what are\",\n\"what's\" : \"what is\",\n\"what've\" : \"what have\",\n\"where's\" : \"where is\",\n\"who'd\" : \"who would\",\n\"who'll\" : \"who will\",\n\"who're\" : \"who are\",\n\"who's\" : \"who is\",\n\"who've\" : \"who have\",\n\"won't\" : \"will not\",\n\"wouldn't\" : \"would not\",\n\"you'd\" : \"you would\",\n\"you'll\" : \"you will\",\n\"you're\" : \"you are\",\n\"you've\" : \"you have\",\n\"'re\": \" are\",\n\"wasn't\": \"was not\",\n\"we'll\":\" will\",\n\"didn't\": \"did not\",\n\"tryin'\":\"trying\"}\n\n\ndef clean_text(x):\n    x = str(x).replace('\\n','')\n    for punct in puncts:\n        x = x.replace(punct, '')\n    return x\n\ndef clean_numbers(x):\n   result = ''.join([i for i in x if not i.isdigit()])\n   return result\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.tokenize.treebank import TreebankWordTokenizer\ntokenizer = TreebankWordTokenizer()\n\ndef handle_contractions(x):\n    x = tokenizer.tokenize(x)\n    return x\n\ndef fix_quote(x):\n    x = [x_[1:] if x_.startswith(\"'\") else x_ for x_ in x]\n    x = ' '.join(x)\n    return x\n\ndef _get_mispell(mispell_dict):\n    mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n    return mispell_dict, mispell_re\n\n\ndef replace_typical_misspell(text):\n    mispellings, mispellings_re = _get_mispell(mispell_dict)\n\n    def replace(match):\n        return mispellings[match.group(0)]\n\n    return mispellings_re.sub(replace, text)\n\n\ndef clean_data(df, columns: list):\n    for col in columns:\n        df[col] = df[col].apply(lambda x: clean_numbers(x))\n        df[col] = df[col].apply(lambda x: clean_text(x.lower())) \n        df[col] = df[col].apply(lambda x: replace_typical_misspell(x))\n        df[col] = df[col].apply(lambda x: handle_contractions(x))  \n        df[col] = df[col].apply(lambda x: fix_quote(x))   \n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ninput_columns = [\n    'comment_text'   \n]\n\n'''applying text cleaning techniques like clean_text,replace_typical_misspell,handle_contractions,fix_quote \non train,test and validation set'''\n\ntrain = clean_data(train, input_columns ) \nval = clean_data(val, input_columns )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head(20)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ninput_columns = [\n    'content'   \n]\ntest_data = clean_data(test_data, input_columns )\n\ndel tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head(20)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Roc-Auc Evaluation metric**"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RocAucEvaluation(Callback):\n    def __init__(self, validation_data=(), interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.X_val, self.y_val = validation_data\n\n        \n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict(self.X_val, verbose=0)\n            score = roc_auc_score(self.y_val, y_pred)\n            print(\"\\n ROC-AUC - epoch: {:d} - score: {:.6f}\".format(epoch+1, score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Tokenization of comments**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Loading XLM-Roberta model tokenizer**"},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL = 'jplu/tf-xlm-roberta-large'\n# First load the real tokenizer\ntokenizer = AutoTokenizer.from_pretrained(MODEL)\n\nsave_path = '/kaggle/working/xlmr_large/'\nif not os.path.exists(save_path):\n    os.makedirs(save_path)\ntokenizer.save_pretrained(save_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Encoding train, validation and test data**"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nx_train = regular_encode(train.comment_text.astype(str), \n                      tokenizer, maxlen=maxlen)\nx_valid = regular_encode(val.comment_text.astype(str).values, \n                      tokenizer, maxlen=maxlen)\nx_test = regular_encode(test_data.content.astype(str).values, \n                     tokenizer, maxlen=maxlen)\n\ny_valid = val.toxic.values\ny_train = train.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_valid","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Training the data with train,validation and test dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Focal Loss"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\ndef focal_loss(gamma=2., alpha=.2):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Building the model and summary check"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, loss='binary_crossentropy', max_len=512):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    x = tf.keras.layers.Dropout(0.3)(cls_token)\n    out = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC()])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = transformers.TFXLMRobertaModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer,loss='binary_crossentropy', max_len=maxlen)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define Define ReduceLROnPlateau callback**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef callback():\n    cb = []\n\n    reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss',  \n                                    factor=0.3, patience=2, \n                                    verbose=1, mode='auto', \n                                    epsilon=0.0001, cooldown=1, min_lr=0.000001)\n    cb.append(reduceLROnPlat)\n    log = CSVLogger('log.csv')\n    cb.append(log)\n\n    RocAuc = RocAucEvaluation(validation_data=(x_valid, y_valid), interval=1)\n    cb.append(RocAuc)\n    \n    return cb\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualization of model architecture"},{"metadata":{"trusted":true},"cell_type":"code","source":"SVG(tf.keras.utils.model_to_dot(model, dpi=80).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Learning rate schedule"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lrfn(lr_start=0.000001, lr_max=0.000002, \n               lr_min=0.0000001, lr_rampup_epochs=7, \n               lr_sustain_epochs=0, lr_exp_decay=.87):\n    lr_max = lr_max * strategy.num_replicas_in_sync\n\n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    \n    return lrfn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(10, 7))\n\nlrfn = build_lrfn()\nplt.plot([i for i in range(35)], [lrfn(i) for i in range(35)]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = 'jigsawMultilingual.hdf5'\nmodel_path1 = '/kaggle/working/jigsawMultilingual.hdf5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n\ncheckpoint = ModelCheckpoint(model_path, monitor='val_accuracy', mode='max', save_best_only=True)\nes = EarlyStopping(monitor='val_accuracy', mode='max', patience=2, \n                   restore_best_weights=True, verbose=1)\nlr_callback = LearningRateScheduler(lrfn, verbose=1)\n\ncallback_list = [checkpoint,  lr_callback]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training"},{"metadata":{},"cell_type":"markdown","source":"Fitting the model with 3 epochs run due to limited availablity of Ram memory"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nN_STEPS = x_train.shape[0] // BATCH_SIZE\nEPOCHS = 3\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=N_STEPS,\n    validation_data=valid_dataset,\n    callbacks=callback_list,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_history_df1 = pd.DataFrame.from_dict(train_history.history)\ntrain_history_df1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(10, 5))\nplt.plot(train_history_df1['val_auc'], label='val')\nplt.plot(train_history_df1['auc'], label='train')\nplt.legend(fontsize=15)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.exists(model_path1):\n    model.load_weights(model_path1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Evaluating the model fit**"},{"metadata":{"trusted":true},"cell_type":"code","source":"score = model.evaluate(x_valid, y_valid, verbose=1)\n\nprint(\"Test Score:\", score[0])\nprint(\"Test Accuracy:\", score[1])   ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"log_dir = \"/kaggle/working/log.csv\"\nif os.path.exists(log_dir):\n    os.remove(log_dir)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Predicting the toxicity - Output"},{"metadata":{"trusted":true},"cell_type":"code","source":"output = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/' + 'sample_submission.csv')\noutput['toxic'] = model.predict(test_dataset, verbose=1)\noutput.to_csv('output_xml.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred1 = model.predict(test_dataset, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred1","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nspl = random.sample(range(len(y_pred1)), 10)\nfor text, sentiment in zip(test.content[spl], y_pred1[spl]):\n    print(sentiment, text)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#  DistilBert Model Implementation"},{"metadata":{},"cell_type":"markdown","source":"Loading the required libraries"},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport re\nimport matplotlib.pyplot as plt\nimport glob","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import string\nfrom sklearn.model_selection import train_test_split, RandomizedSearchCV, GridSearchCV, StratifiedKFold\n\n\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.pipeline import Pipeline\nfrom gensim import utils\nimport gensim.parsing.preprocessing as gsp\n\nfrom sklearn.model_selection import cross_val_score\nfrom sklearn.metrics import accuracy_score, f1_score, roc_auc_score\n\nimport seaborn as sns\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Dropout, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping, ModelCheckpoint\nfrom kaggle_datasets import KaggleDatasets\nimport transformers\nfrom tqdm.notebook import tqdm\nimport tokenizers\nfrom tokenizers import BertWordPieceTokenizer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Loading the required dataset from .CSV files**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading train data\nneeded_cols = ['toxic', 'comment_text']\n\ntranslated_train_files = glob.glob('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-*-cleaned.csv')\ntranslated_train_dfs = []\nfor filename in translated_train_files:\n    df = pd.read_csv(filename, usecols=needed_cols)\n    lang = re.findall('train-google-(.*)-cleaned.csv', filename)[0]\n    df['lang'] = lang\n    translated_train_dfs.append(df)\n\ntrain_en = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=needed_cols)\ntrain_en['lang'] = 'en'\n\ntranslated_train_dfs.append(train_en)\n    \ntrain_df = pd.concat(translated_train_dfs).sample(n=500000).reset_index(drop=True) #Training dataset\n\ny_train = train_df['toxic'].values\n\n\ndel df, translated_train_dfs, train_en\ngc.collect()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(train_df))\ntrain_df.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Preprocessing"},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(text):\n    text = str(text)\n    text = re.sub(r'[0-9\"]', '', text) # number\n    text = re.sub(r'#[\\S]+\\b', '', text) # hash\n    text = re.sub(r'@[\\S]+\\b', '', text) # mention\n    text = re.sub(r'https?\\S+', '', text) # link\n    text = re.sub(r'\\s+', ' ', text) # multiple white spaces\n#     text = re.sub(r'\\W+', ' ', text) # non-alphanumeric\n    return text.strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def text_process(text):\n    ws = text.split(' ')\n    if(len(ws)>160):\n        text = ' '.join(ws[:160]) + ' ' + ' '.join(ws[-32:])\n    return text\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Loading DistilBert Tokenizer**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# First load the real tokenizer\ntokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\n\nsave_path = '/kaggle/working/distilbert_base_uncased/'\nif not os.path.exists(save_path):\n    os.makedirs(save_path)\ntokenizer.save_pretrained(save_path)\n\nfast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', lowercase=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Encoding of comments**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n \n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def len_sent(data):\n    return len(data.split())\ntrain_df[\"num_words_comment_text\"] = train_df[\"comment_text\"].apply(lambda x : len_sent(x))\nsns.kdeplot(train_df[train_df[\"toxic\"] == 0][\"num_words_comment_text\"].values, shade = True, color = \"red\", label='non_toxity')\nsns.kdeplot(train_df[train_df[\"toxic\"] == 1][\"num_words_comment_text\"].values, shade = True, color = \"blue\", label='toxity')\n\ndel train_df['toxic']; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Configuration setup**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nAUTO = tf.data.experimental.AUTOTUNE\nSHUFFLE = 2048\nEPOCHS1 = 20\nEPOCHS2 = 4\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nMAX_LEN = 192\nVERBOSE = 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Detect hardware to return appropriate distribution strategy**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Encoding train data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df['comment_text'] = train_df['comment_text'].apply(lambda x: clean_text(x))\ntrain_df['comment_text'] = train_df['comment_text'].apply(lambda x: text_process(x))\nx_train = fast_encode(train_df['comment_text'].astype(str), fast_tokenizer, maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Loading and cleaning validation dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\nvalid['comment_text'] = valid.apply(lambda x: clean_text(x['comment_text']), axis=1)\nvalid['comment_text'] = valid['comment_text'].apply(lambda x: text_process(x))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Encoding validation dataset"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_valid = fast_encode(valid['comment_text'].astype(str), fast_tokenizer, maxlen=MAX_LEN)\ny_valid = valid['toxic'].values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Build dataset objects"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(SHUFFLE)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Call Backs**"},{"metadata":{"trusted":true},"cell_type":"code","source":"lrs = ReduceLROnPlateau(monitor='val_auc', mode ='max', factor = 0.7, min_lr= 1e-7, verbose = 1, patience = 2)\nes1 = EarlyStopping(monitor='val_auc', mode='max', verbose = 1, patience = 5, restore_best_weights=True)\nes2 = EarlyStopping(monitor='auc', mode='max', verbose = 1, patience = 1, restore_best_weights=True)\ncallbacks_list1 = [lrs,es1]\ncallbacks_list2 = [lrs,es2]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Build model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    x = tf.keras.layers.Dropout(0.4)(cls_token)\n    out = Dense(1, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC(name='auc'), 'accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Loading model into TPU**"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-multilingual-cased')\n    )\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Training the model**"},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = len(y_train) // (BATCH_SIZE*8)\n\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS1,\n    callbacks=callbacks_list1,\n    verbose=VERBOSE\n)\n\ndel train_dataset; gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_history_df = pd.DataFrame.from_dict(train_history.history)\ntrain_history_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.figure(figsize=(10, 5))\nplt.plot(train_history_df['val_auc'], label='val')\nplt.plot(train_history_df['auc'], label='train')\nplt.legend(fontsize=15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Evaluating the model fit**"},{"metadata":{"trusted":true},"cell_type":"code","source":"score = model.evaluate(x_valid,y_valid, verbose=1)\n\nprint(\"Test Score:\", score[0])\nprint(\"Test Accuracy:\", score[1])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Test data toxic prediction**"},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\ntest['content'] = test.apply(lambda x: clean_text(x['content']), axis=1)\ntest['content'] = test['content'].apply(lambda x: text_process(x))\nx_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)\n\noutput_distil = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output_distil['toxic'] = model.predict(test_dataset, verbose=1)\noutput_distil['toxic'].hist(bins=100, log=False, alpha=1)\noutput_distil.to_csv('output_distilbert.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output_distil","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred = model.predict(test_dataset, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_pred","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import random\nspl = random.sample(range(len(y_pred)), 10)\nfor text, sentiment in zip(test.content[spl], y_pred[spl]):\n    print(sentiment, text)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}