{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"! pip install googletrans","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport os\nimport tensorflow as tf\nimport matplotlib.pyplot as plt\nfrom matplotlib.pyplot import *\nfrom tqdm import tqdm\nimport nltk\nimport string\nimport seaborn as sns\nimport re\nimport nltk\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.metrics import precision_score, recall_score, roc_curve\nfrom keras import layers\nfrom keras import models\nfrom keras import layers\nfrom keras import *\n\nimport transformers\nfrom tensorflow.keras.callbacks import Callback\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\n\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom kaggle_datasets import KaggleDatasets\nimport transformers\n\nfrom tokenizers import BertWordPieceTokenizer\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport seaborn as sns\nimport os\nimport re\n\nimport transformers\nimport tensorflow as tf\nfrom tqdm.notebook import tqdm\nfrom wordcloud import WordCloud, STOPWORDS\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tokenizers import BertWordPieceTokenizer\nfrom kaggle_datasets import KaggleDatasets\nfrom googletrans import Translator\nfrom colorama import Fore, Back, Style, init\nimport plotly.graph_objects as go\ntranslator = Translator()\n\nfrom tensorflow.keras.layers import (Dense, Input, LSTM, Bidirectional, Activation, Conv1D, \n                                     GRU,Embedding, Flatten, Dropout, Add, concatenate, MaxPooling1D,\n                                     GlobalAveragePooling1D,  GlobalMaxPooling1D, \n                                     GlobalMaxPool1D,SpatialDropout1D)\n\nfrom tensorflow.keras import (initializers, regularizers, constraints, \n                              optimizers, layers, callbacks)\n\nsns.set(style=\"darkgrid\")\n\n\n\n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\ntest = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/test.csv\")\nsubm = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['comment_text']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#nltk.download()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# What methods and attributes are in the nltk\ndir(nltk)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Did it work ?\nfrom nltk.corpus import stopwords\n\nstopwords.words('english')[0:10]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read in view the raw data\n\ntrain = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Drop unused columns and label\ntrain = train.drop(labels = ['id','severe_toxic','obscene','threat','insult','identity_hate'], axis=1)\ntrain.columns = ['comment','toxic']\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# How big is the dataset\ntrain.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# What portion of our comment are actually toxic ?\nprint(train['toxic'].value_counts())\n\n# A toxic comment would receive a 1.0.\n# A non-toxic comment would receive a 0.0.\nsns.countplot(train.toxic)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"202165 comments are Non-toxic\n\n21384  comments are Toxic"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Are we missing any data ?\nprint(\"Number of nulls toxic : {}\".format(train['toxic'].isnull().sum()))\nprint(\"Number of nulls comment : {}\".format(train['comment'].isnull().sum()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Read in raw data and clean up the columns\npd.set_option('display.max_colwidth', 100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# What punctuation is included in the dafault list ?\nstring.punctuation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Split the data into training and test set"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Split the data into training and test set\nX_train, X_test, y_train, y_test = train_test_split(train['comment'], train['toxic'], test_size = 0.2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare Data For Modeling"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Prepare Data For Modeling\n# Import the tools we will need from Keras\n\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Initalize and fit the tokenizer\ntokenizer =  Tokenizer()\ntokenizer.fit_on_texts(X_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Use that tokenizer to transforme the comment in the training and test sets\n\nX_train_seq = tokenizer.texts_to_sequences(X_train)\nX_test_seq = tokenizer.texts_to_sequences(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# What do these sequences look like ?\nX_train_seq[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pad the sequences so each sequence is the same length\nX_train_seq_padded = pad_sequences(X_train_seq, 50)\nX_test_seq_padded = pad_sequences(X_test_seq, 50)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# What do these padded sequences look like ?\nX_train_seq_padded[0]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Buil The Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Buil The Model\n# Import the tools needed from keras and functions to calculate recall and precision\n\nimport keras.backend as K\nfrom keras.layers import Dense, Embedding, LSTM\nfrom keras.models import Sequential\nfrom sklearn import metrics\n\ndef recall_m(y_true, y_pred):\n    true_positives  = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    possibles_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    recall = true_positives / (possibles_positives + K.epsilon())\n    return recall\n\ndef precision_m(y_true, y_pred):\n    true_positives  = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n    precision = true_positives / (predicted_positives + K.epsilon())\n    return precision\n\ndef roc_auc(predictions, target):\n    tg, pr, thsld = metrics.roc_curve(target, predictions)\n    roc_auc = metrics.auc(tg, pr)\n    return roc_auc\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Construct a simple RNN Model\n\nmodel =  Sequential()\nmodel.add(Embedding(len(tokenizer.index_word) + 1, 32))\nmodel.add(LSTM(32, dropout = 0, recurrent_dropout = 0))\nmodel.add(Dense(32, activation = 'relu'))\nmodel.add(Dense(1, activation = 'sigmoid'))\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compile The Model\nmodel.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy', precision_m, recall_m])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Fit The RNN Model\nhistory = model.fit(X_train_seq_padded, y_train, batch_size=32, epochs=10, validation_data=(X_test_seq_padded, y_test))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Plot the evaluation metrics by epoch for the model to see if we are over or underfitting\n\n\nfor i in ['accuracy', 'precision_m', 'recall_m']:\n    acc = history.history[i]\n    val_acc = history.history['val_{}'.format(i)]\n    epochs = range(1, len(acc) + 1)\n    \n    plt.figure()\n    plt.plot(epochs, acc, label='Training Accuracy')\n    plt,plot(epochs, val_acc, label='Test Accuracy')\n    plt.title('Results for {}'.format(i))\n    plt.legend()\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = model.predict(X_test_seq_padded)\nprint(\"Auc: %.2f%%\" % (roc_auc(scores, y_test)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Using"},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n    \"\"\"\n    Encoder for encoding the text into sequence of integers for BERT Input\n    \"\"\"\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#IMP DATA FOR CONFIG\n\nAUTO = tf.data.experimental.AUTOTUNE\n\n\n# Configuration\nEPOCHS = 3\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nMAX_LEN = 192","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# First load the real tokenizer\ntokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\n# Save the loaded tokenizer locally\ntokenizer.save_pretrained('.')\n# Reload it with the huggingface tokenizers library\nfast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\nfast_tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = fast_encode(train1.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=MAX_LEN)\nx_test = fast_encode(test.content.astype(str), fast_tokenizer, maxlen=MAX_LEN)\n\ny_train = train1.toxic.values\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    \"\"\"\n    function for training the BERT model\n    \"\"\"\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    out = Dense(1, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-multilingual-cased')\n    )\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_train.shape[0] // BATCH_SIZE\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=EPOCHS*2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subResult = pd.read_csv('./submission.csv')\nsubResult.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Deep Learning"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_set1 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain_set2 = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\ntrain_set2.toxic = train_set2.toxic.round().astype(int)\nvalid = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Combine train1 with a subset of train2\ntrain = pd.concat([\n    train_set1[['comment_text', 'toxic']],\n    train_set2[['comment_text', 'toxic']].query('toxic==1'),\n    train_set2[['comment_text', 'toxic']].query('toxic==0').sample(n=100000, random_state=0)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid.shape)\nvalid.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Columns"},{"metadata":{},"cell_type":"markdown","source":"id - identifier within each file.\ncomment_text - the text of the comment to be classified.\nlang - the language of the comment.\ntoxic - whether or not the comment is classified as toxic. (Does not exist in test.csv.)"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.toxic.value_counts())\nsns.countplot(train.toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid.toxic.value_counts())\nsns.countplot(valid.toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid.lang.value_counts())\nsns.countplot(valid.lang)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.lang.value_counts())\nsns.countplot(test.lang)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Distribution of Characters & Words"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ax(rows=1, cols=2, size=7):\n    \"\"\"Return a Matplotlib Axes array to be used in\n    all visualizations in the notebook. Provide a\n    central point to control graph sizes.\n    \n    Adjust the size attribute to control how big to render images\n    \"\"\"\n    fig, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n    return fig, ax","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = get_ax()\n\nsns.distplot(train[train['toxic']==0]['comment_text'].str.len(), axlabel=\"Non Toxic\", ax=ax[0])\nsns.distplot(train[train['toxic']==0]['comment_text'].str.split().str.len(), axlabel=\"Non Toxic\", ax=ax[1])\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = get_ax()\n\nsns.distplot(train[train['toxic']==1]['comment_text'].str.len(), axlabel=\"Toxic\", ax=ax[0])\nsns.distplot(train[train['toxic']==1]['comment_text'].str.split().str.len(), axlabel=\"Toxic\", ax=ax[1])\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Wordclouds - Frequent Words"},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = set(STOPWORDS)\n\ndef show_wordcloud(data, title = None):\n    wordcloud = WordCloud(\n        background_color='white',\n        stopwords=stopwords,\n        max_words=50,\n        max_font_size=40, \n        scale=5,\n        random_state=1\n    ).generate(str(data))\n\n    fig = plt.figure(1, figsize=(10,10))\n    plt.axis('off')\n    if title: \n        fig.suptitle(title, fontsize=20)\n        fig.subplots_adjust(top=2.3)\n\n    plt.imshow(wordcloud)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_wordcloud(train['comment_text'].sample(20000), \n               title = '[Comment_Text] Prevalent Words')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_wordcloud(valid['comment_text'].sample(1000), \n               title = '[Comment_Text] Prevalent Words')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_wordcloud(test['content'].sample(1000), \n               title = '[Content] Prevalent Words')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Comments"},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(5):\n    print(f'[CONTENT {i}]\\n', train['comment_text'][i])\n    print()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Modeling"},{"metadata":{},"cell_type":"markdown","source":"Here I will be following xhlulu approach. Appreciate his effort if you his notebook."},{"metadata":{"trusted":true},"cell_type":"code","source":"# fast encoder\ndef fast_encode(texts, tokenizer, chunk_size=240, maxlen=512):\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in range(0, len(texts), chunk_size):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# general encoder\ndef regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"TPU Configs"},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Create strategy from tpu\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\n# Configuration\nEPOCHS = 4\nBATCH_SIZE = 16* strategy.num_replicas_in_sync\nMODEL = 'jplu/tf-xlm-roberta-large'\nMAX_LEN = 224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://huggingface.co/models\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\n\n#First load the real tokenizer\ntokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Create fast tokenizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nx_train = regular_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)\nx_valid = regular_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)\nx_test = regular_encode(test.content.values, tokenizer, maxlen=MAX_LEN)\n\ny_train = train.toxic.values\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Build datasets objects"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We will use focal loss as our loss function"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\ndef label_smoothing(y_true,y_pred):\n     return tf.keras.losses.binary_crossentropy(y_true,y_pred,label_smoothing=0.15)\n\ndef focal_loss(gamma=2., alpha=.2):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Load model into the TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    \n    cls_token = sequence_output[:, 0, :]\n    x = Dropout(0.3)(cls_token)\n    out = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss=label_smoothing,\n                  metrics=[tf.keras.metrics.AUC()]) # competition metrics\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow.keras.backend as K\n\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer, max_len=MAX_LEN)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import SVG\nSVG(tf.keras.utils.model_to_dot(model, dpi=70).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def callback():\n    cb = []\n    \"\"\"\n    Model-Checkpoint\n    \"\"\"\n    checkpoint = callbacks.ModelCheckpoint('model.h5',\n                                       save_best_only=True, \n                                       mode='min',\n                                       monitor='val_loss', #  \n                                       save_weights_only=True, verbose=0)\n\n    cb.append(checkpoint)\n    \n    # Callback that streams epoch results to a csv file.\n    log = callbacks.CSVLogger('log.csv')\n    cb.append(log)\n\n    return cb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"calls = callback()\nn_steps = x_train.shape[0] // BATCH_SIZE\n\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    callbacks = calls,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Visualize the Model Performances"},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_model_preds(model, indices=[0, 17, 1, 24]):\n    comments = valid.comment_text.loc[indices].values.tolist()\n    preds = model.predict(x_valid[indices].reshape(len(indices), -1))\n\n    for idx, i in enumerate(indices):\n        if y_valid[i] == 0:\n            label = \"Non-toxic\"\n            color = f'{Fore.GREEN}'\n            symbol = '\\u2714'\n        else:\n            label = \"Toxic\"\n            color = f'{Fore.RED}'\n            symbol = '\\u2716'\n\n        print('{}{} {}'.format(color, str(idx+1) + \". \" + label, symbol))\n        print(f'{Style.RESET_ALL}')\n        print(\"ORIGINAL\")\n        print(comments[idx]); print(\"\")\n        print(\"TRANSLATED\")\n        print(translator.translate(comments[idx]).text)\n        fig = go.Figure()\n        if list.index(sorted(preds[:, 0]), preds[idx][0]) > 1:\n            yl = [preds[idx][0], 1 - preds[idx][0]]\n        else:\n            yl = [1 - preds[idx][0], preds[idx][0]]\n        fig.add_trace(go.Bar(x=['Non-Toxic', 'Toxic'], y=yl, marker=dict(color=[\"seagreen\", \"indianred\"])))\n        fig.update_traces(name=comments[idx])\n        fig.update_layout(xaxis_title=\"Labels\", yaxis_title=\"Probability\", template=\"plotly_white\", title_text=\"Predictions for validation comment #{}\".format(idx+1))\n        fig.show()\n        \nvisualize_model_preds(model)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"subResult = pd.read_csv('./submission.csv')\nsubResult.head()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}