{"cells":[{"metadata":{},"cell_type":"markdown","source":"<center><img src='https://raw.githubusercontent.com/dimitreOliveira/MachineLearning/master/Kaggle/Jigsaw%20Multilingual%20Toxic%20Comment%20Classification/banner.png'></center>\n\n<br>\n<center><h1>Jigsaw Multilingual Toxic Comment Classification</h1></center>\n\n<br>\n<center><h3>Jigsaw Classification - DistilBERT with TPU and TF</h3></center>\n\n<br>\n<br>\nIt only takes one toxic comment to sour an online discussion. The Conversation AI team, a research initiative founded by [Jigsaw](https://jigsaw.google.com/) and Google, builds technology to protect voices in conversation. A main area of focus is machine learning models that can identify toxicity in online conversations, where toxicity is defined as anything rude, disrespectful or otherwise likely to make someone leave a discussion. If these toxic contributions can be identified, we could have a safer, more collaborative internet."},{"metadata":{},"cell_type":"markdown","source":"## Dependencies"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import os,gc , warnings, transformers\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nfrom matplotlib import pyplot as plt\nfrom tokenizers import BertWordPieceTokenizer\nimport tensorflow as tf\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import optimizers, metrics, losses\nfrom tensorflow.keras.layers import Dense, Input, Dropout, GlobalAveragePooling1D\nfrom tensorflow.keras.callbacks import LearningRateScheduler, EarlyStopping\nfrom sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve, auc, classification_report\n\ndef seed_everything(seed=0):\n    np.random.seed(seed)\n    tf.random.set_seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    os.environ['TF_DETERMINISTIC_OPS'] = '1'\n\nseed = 0\nseed_everything(seed)\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# Auxiliary functions\ndef plot_metrics(history, metric_list):\n    fig, axes = plt.subplots(len(metric_list), 1, sharex='col', figsize=(20, 18))\n    axes = axes.flatten()\n    \n    for index, metric in enumerate(metric_list):\n        axes[index].plot(history[metric], label='Train %s' % metric)\n        axes[index].plot(history['val_%s' % metric], label='Validation %s' % metric)\n        axes[index].legend(loc='best', fontsize=16)\n        axes[index].set_title(metric)\n\n    plt.xlabel('Epochs', fontsize=16)\n    sns.despine()\n    plt.show()\n    \ndef plot_aur_curve(y_train, train_pred, y_valid, valid_pred):\n    fpr_train, tpr_train, _ = roc_curve(y_train, train_pred)\n    roc_auc_train = auc(fpr_train, tpr_train)\n    fpr_valid, tpr_valid, _ = roc_curve(y_valid, valid_pred)\n    roc_auc_valid = auc(fpr_valid, tpr_valid)\n\n    fig, ax = plt.subplots(1, 1, figsize=(8, 8))\n    plt.title('Receiver Operating Characteristic')\n    plt.plot(fpr_train, tpr_train, color='blue', label='Train AUC = %0.2f' % roc_auc_train)\n    plt.plot(fpr_valid, tpr_valid, color='purple', label='ValidationAUC = %0.2f' % roc_auc_valid)\n    plt.legend(loc = 'lower right')\n    plt.plot([0, 1], [0, 1],'r--')\n    plt.xlim([0, 1])\n    plt.ylim([0, 1])\n    plt.ylabel('True Positive Rate')\n    plt.xlabel('False Positive Rate')\n    plt.show()\n    \ndef plot_confusion_matrix(y_train, train_pred, y_valid, valid_pred, labels=[0, 1]):\n    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(20, 8))\n    train_cnf_matrix = confusion_matrix(y_train, train_pred)\n    validation_cnf_matrix = confusion_matrix(y_valid, valid_pred)\n\n    train_cnf_matrix_norm = train_cnf_matrix.astype('float') / train_cnf_matrix.sum(axis=1)[:, np.newaxis]\n    validation_cnf_matrix_norm = validation_cnf_matrix.astype('float') / validation_cnf_matrix.sum(axis=1)[:, np.newaxis]\n\n    train_df_cm = pd.DataFrame(train_cnf_matrix_norm, index=labels, columns=labels)\n    validation_df_cm = pd.DataFrame(validation_cnf_matrix_norm, index=labels, columns=labels)\n\n    sns.heatmap(train_df_cm, annot=True, fmt='.2f', cmap=\"Blues\",ax=ax1).set_title('Train')\n    sns.heatmap(validation_df_cm, annot=True, fmt='.2f', cmap=sns.cubehelix_palette(8),ax=ax2).set_title('Validation')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU configuration"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_toxic = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", \n                          usecols=['id', 'comment_text', 'toxic'])\ntrain_bias = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\", \n                         usecols=['id', 'comment_text', 'toxic'])\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv', \n                    usecols=['id', 'comment_text', 'toxic'])\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv', \n                   usecols=['id', 'content'])\n\nprint('Jigsaw toxic comment samples %d' % len(train_toxic))\ndisplay(train_toxic.head())\nprint('Jigsaw unintended bias samples %d' % len(train_bias))\ndisplay(train_bias.head())","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model parameters"},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_LEN = 512\nBATCH_SIZE = 64 * strategy.num_replicas_in_sync\nEPOCHS = 12\nLEARNING_RATE = 1e-5 # * strategy.num_replicas_in_sync\nES_PATIENCE = 3\n\nbase_model_name = 'distilbert-base-multilingual-cased'\nmodel_path = 'model.h5'\nvocab_path = '/kaggle/working'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tokenizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.DistilBertTokenizer.from_pretrained(base_model_name)\ntokenizer.save_pretrained(vocab_path)\n\ntokenizer = BertWordPieceTokenizer(vocab_path + '/vocab.txt', lowercase=False)\ntokenizer.enable_truncation(max_length=MAX_LEN)\ntokenizer.enable_padding(max_length=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build TF datasets"},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = [x.ids for x in tokenizer.encode_batch(train_toxic['comment_text'].apply(lambda x : x).tolist())]\nx_valid = [x.ids for x in tokenizer.encode_batch(valid['comment_text'].apply(lambda x : x).tolist())]\nx_test = [x.ids for x in tokenizer.encode_batch(test['content'].apply(lambda x : x).tolist())]\n\ny_train = train_toxic['toxic'].values\ny_valid = valid['toxic'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ndef get_training_dataset():\n    dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train))\n    dataset = dataset.repeat()\n    dataset = dataset.shuffle(2048)\n    dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_validation_dataset():\n    dataset = tf.data.Dataset.from_tensor_slices((x_valid, y_valid))\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.cache()\n    dataset = dataset.prefetch(AUTO)\n    return dataset\n\ndef get_test_dataset():\n    dataset = tf.data.Dataset.from_tensor_slices(x_test)\n    dataset = dataset.batch(BATCH_SIZE)\n    dataset = dataset.prefetch(AUTO)\n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Learning rate schedule"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"del train_bias\ngc.collect()\n\nLR_START = 1e-7\nLR_MIN = 1e-6\nLR_MAX = LEARNING_RATE\nLR_RAMPUP_EPOCHS = 2\nLR_SUSTAIN_EPOCHS = 0\nLR_EXP_DECAY = .6\n\ndef lrfn(epoch):\n    if epoch < LR_RAMPUP_EPOCHS:\n        lr = (LR_MAX - LR_START) / LR_RAMPUP_EPOCHS * epoch + LR_START\n    elif epoch < LR_RAMPUP_EPOCHS + LR_SUSTAIN_EPOCHS:\n        lr = LR_MAX\n    else:\n        lr = (LR_MAX - LR_MIN) * LR_EXP_DECAY**(epoch - LR_RAMPUP_EPOCHS - LR_SUSTAIN_EPOCHS) + LR_MIN\n    return lr\n    \nrng = [i for i in range(EPOCHS)]\ny = [lrfn(x) for x in rng]\n\nfig, ax = plt.subplots(figsize=(20, 8))\nplt.plot(rng, y)\nprint(\"Learning rate schedule: {:.3g} to {:.3g} to {:.3g}\".format(y[0], max(y), y[-1]))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def model_fn():\n    base_model = transformers.TFDistilBertModel.from_pretrained(base_model_name)\n    input_word_ids = Input(shape=(MAX_LEN,), dtype=tf.int32, name='input_word_ids')\n    sequence_output = base_model(input_word_ids)[0]\n    \n    x = GlobalAveragePooling1D()(sequence_output)\n    x = Dropout(0.25)(x)\n    output = Dense(1, activation='sigmoid', name='output')(x)\n    \n    model = Model(inputs=input_word_ids, outputs=output)\n    model.compile(optimizers.Adam(lr=LEARNING_RATE), \n                  loss=losses.BinaryCrossentropy(), \n                  metrics=['accuracy', metrics.AUC()])\n    \n    return model\n\nwith strategy.scope():\n    model = model_fn()\n    \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train"},{"metadata":{"trusted":true},"cell_type":"code","source":"STEPS_PER_EPOCH = len(x_train) // BATCH_SIZE\n\nes = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, \n                   restore_best_weights=True, verbose=1)\nlr_callback = LearningRateScheduler(lrfn, verbose=1)\n\n\nhistory = model.fit(x=get_training_dataset(),\n                    steps_per_epoch=STEPS_PER_EPOCH,\n                    validation_data=get_validation_dataset(),\n                    callbacks=[es, lr_callback],\n                    epochs=EPOCHS, \n                    verbose=1).history","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model loss graph"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"sns.set(style=\"whitegrid\")\nplot_metrics(history, metric_list=['loss', 'accuracy', 'auc'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model evaluation"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"train_pred = model.predict(get_training_dataset(), steps=STEPS_PER_EPOCH)\nvalid_pred = model.predict(get_validation_dataset())\ntrain_toxic = train_toxic[:len(train_pred)]\ntrain_toxic['pred'] = train_pred\nvalid['pred'] = valid_pred\n\nprint('Train set ROC AUC %.4f' % roc_auc_score(train_toxic['toxic'], train_toxic['pred']))\nprint(classification_report(train_toxic['toxic'],  np.round(train_toxic['pred'])))\nprint('Validation set ROC AUC %.4f' % roc_auc_score(valid['toxic'], valid['pred']))\nprint(classification_report(valid['toxic'],  np.round(valid['pred'])))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# ROC Curve"},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"plot_aur_curve(train_toxic['toxic'], train_toxic['pred'], valid['toxic'], valid['pred'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Confusion matrix"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"plot_confusion_matrix(train_toxic['toxic'], np.round(train_toxic['pred']), \n                      valid['toxic'], np.round(valid['pred']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize predictions"},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"print('Train set')\ndisplay(train_toxic[['comment_text', 'toxic', 'pred']].head(10))\nprint('Validation set')\ndisplay(valid[['comment_text', 'toxic', 'pred']].head(10))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Test set predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"Y_test = model.predict(get_test_dataset())\nsubmission = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nsubmission['toxic'] = Y_test\nsubmission.to_csv('submission.csv', index=False)\nsubmission.head(10)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}