{"cells":[{"metadata":{},"cell_type":"markdown","source":"## About this notebook\n\n*[Jigsaw Multilingual Toxic Comment Classification](https://www.kaggle.com/c/jigsaw-multilingual-toxic-comment-classification)* is the 3rd annual competition organized by the Jigsaw team. It follows *[Toxic Comment Classification Challenge](https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge)*, the original 2018 competition, and *[Jigsaw Unintended Bias in Toxicity Classification](https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification)*, which required the participants to consider biased ML predictions in their new models.    \n\n**This year**, the goal is to use english only training data to run toxicity predictions on many different languages, which can be done using multilingual models, and speed up using TPUs.\n\nThis notebook instead aims at constructing a **fast, concise, reusable, and beginner-friendly model scaffold**. \n\n**THIS DOES NOT USE ANY TRANSLATED DATA(other langs--->English lang), BUT IT DOES TRAIN ON THE VALIDATION SET.**\n\n\n### References\n* Original Author: [@xhlulu](https://www.kaggle.com/xhlulu/)\n* Original notebook: [Link](https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras)","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, Dropout, BatchNormalization, Activation\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom kaggle_datasets import KaggleDatasets\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n#import transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom transformers import (\n    AdamW, get_linear_schedule_with_warmup, get_constant_schedule, \n    XLMRobertaTokenizer, XLMRobertaModel, XLMRobertaConfig,\n)\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Data access\n#GCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\n# Configuration\nEPOCHS = 5\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\nMAX_LEN = 128  #------------------------------------ changed\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"publictrain = pd.read_csv(\"/kaggle/input/jigsaw-public-baseline-train-data/train_data.csv\")\n#stratified sampling to get good proportion of data: 2*sample_size from each language class\n# Why 2?: 1 for each lang + 1 for each of the two toxic classes\nsample_size = 20000\ndf_train = publictrain.groupby(['lang','toxic'], group_keys=False).apply(lambda x: x.sample(min(len(x),sample_size)))\ndf_train = df_train.sample(frac=1).reset_index(drop=True)#shuffling\ndf_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# cols = list(set(train2.columns).intersection(train1.columns))\n# #['id', 'comment_text', 'toxic', 'obscene', 'threat','insult']#common\n# print(\"for toxic comment competition 2018\")\n# print(train1[cols].isna().sum()/len(train1))\n# print(\"-\"*20)\n# print(\"for unintended bias competition 2019\")\n# print(train2[cols].isna().sum()/len(train2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Since the common columns donot have missing values we might as well use them during training","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Load text data into memory","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\", usecols=[\"comment_text\", \"toxic\"])\n# train2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\", usecols=[\"comment_text\", \"toxic\"])\n# train2.toxic = train2.toxic.round().astype(int)\n# df_train = pd.concat([\n#     train1[['comment_text', 'toxic']],\n#     train2[['comment_text', 'toxic']].query('toxic==1'),\n#     train2[['comment_text', 'toxic']].query('toxic==0').sample(n=99937, random_state=0),])\n# df_train = df_train.sample(frac=1).reset_index(drop=True)#shuffling\n# import gc\n# del train1, train2\n# gc.collect(); gc.collect();\n# print(df_train.shape, df_valid.shape)\n# gc.collect(); gc.collect(); gc.collect();\n\n\ndf_valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ndf_valid = df_valid.sample(frac=1).reset_index(drop=True)#shuffling\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom joblib import Parallel, delayed\ntokenizer = AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-base')\n\ndef regular_encode(row, is_test=False, maxlen=MAX_LEN):\n    outp = None\n    if is_test:#for test data\n        enc_di = tokenizer.encode_plus(\n            str(row),#row:text\n            return_attention_masks=False, \n            return_token_type_ids=False,\n            pad_to_max_length=True,\n            max_length=maxlen\n        )\n        outp = np.array(enc_di['input_ids'])\n    else:#for validation/train data\n        enc_di = tokenizer.encode_plus(\n            str(row[0]),#row:(text,label)\n            return_attention_masks=False, \n            return_token_type_ids=False,\n            pad_to_max_length=True,\n            max_length=maxlen\n        )\n        outp = np.array(enc_di['input_ids']), row[1]\n    return outp\n                        \nrows = zip(df_train['comment_text'].values.tolist(), df_train.toxic.values.tolist())\ntrain = Parallel(n_jobs=4, backend='multiprocessing')(delayed(regular_encode)(row) for row in tqdm(rows))\n\nrows = zip(df_valid['comment_text'].values.tolist(), df_valid.toxic.values.tolist())\nvalid = Parallel(n_jobs=4, backend='multiprocessing')(delayed(regular_encode)(row) for row in tqdm(rows))\n                        \nrows = test.content.values.tolist()\nx_test = Parallel(n_jobs=4, backend='multiprocessing')(delayed(regular_encode)(row,is_test=True) for row in tqdm(rows))\n\nx_train = np.vstack(np.array(train)[:,0])\ny_train = np.array(train)[:,1].astype(np.int32)\nx_valid = np.vstack(np.array(valid)[:,0])\ny_valid = np.array(valid)[:,1].astype(np.int32)\nx_train.shape,y_train.shape,x_valid.shape,y_valid.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build tf.datasets objects","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_dataset = (\n#     tf.data.Dataset\n#     .from_tensor_slices((x_train,y_train))\n#     .repeat()\n#     .shuffle(2048)\n#     .batch(BATCH_SIZE)\n#     .prefetch(AUTO)\n# )#tensorflow.python.data.ops.dataset_ops.PrefetchDataset\n\n# valid_dataset = (\n#     tf.data.Dataset\n#     .from_tensor_slices((x_valid,y_valid))\n#     .batch(BATCH_SIZE)\n#     .cache()\n#     .prefetch(AUTO)\n# )#tensorflow.python.data.ops.dataset_ops.PrefetchDataset\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)#tensorflow.python.data.ops.dataset_ops.PrefetchDataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"valid--->Counter({'es-panish': 2500, 'it-alian': 2500, 'tr-turkish': 3000})  \ntest----->Counter({'tr': 14000,'ru': 10948,'it': 8494,'fr': 10920,'pt': 11012,'es': 8438})","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## Build model & compile it","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras.utils.generic_utils import get_custom_objects\ndef gelu(x):\n    \"\"\"Gaussian Error Linear Unit.\n    This is a smoother version of the RELU.\n    Original paper: https://arxiv.org/abs/1606.08415\n    Args:\n        x: float Tensor to perform activation.\n    Returns:\n        `x` with the GELU activation applied.\n    \"\"\"\n    cdf = 0.5 * (1.0 + tf.tanh((np.sqrt(2 / np.pi) * (x + 0.044715 * tf.pow(x, 3)))))\n    return x * cdf#`x` with the GELU activation applied\nget_custom_objects().update({'gelu': Activation(gelu)})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]#cls_token is a vector of length 768 marginalised against other 2 dimensions\n    x = Dense(16, activation=gelu)(cls_token)\n    x = BatchNormalization()(x)\n    x = Dropout(0.3)(x)\n    out = Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])\n    return model\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load model into TPU","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained('jplu/tf-xlm-roberta-base')\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"First, we train on the subset of the training set, which is completely in English.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"K = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train(X_train,y_train):\n    oof_predictions = []\n    from sklearn.model_selection import KFold\n    from tensorflow.keras.callbacks import LearningRateScheduler\n    import math\n    kf = KFold(n_splits=K, random_state=1, shuffle=True)\n    lr_schedule = LearningRateScheduler(lambda epoch: 0.001 * math.pow(0.001, math.floor((1+epoch)/3.0)))\n    \n    for ind, (tr, val) in enumerate(kf.split(X_train)):\n        X_tr = X_train[tr]\n        y_tr = y_train[tr]\n        X_vl = X_train[val]\n        y_vl = y_train[val]\n        print(X_tr.shape,y_tr.shape,X_vl.shape,y_vl.shape)\n        \n        train_dataset = (\n            tf.data.Dataset\n            .from_tensor_slices((X_tr,y_tr))\n            .repeat()\n            .shuffle(2048)\n            .batch(BATCH_SIZE)\n            .prefetch(AUTO)\n        )\n\n        valid_dataset = (\n            tf.data.Dataset\n            .from_tensor_slices((X_vl,y_vl))\n            .batch(BATCH_SIZE)\n            .cache()\n            .prefetch(AUTO)\n        )\n        \n        n_steps = X_tr.shape[0] // BATCH_SIZE\n        train_history = model.fit(\n                        train_dataset,\n                        steps_per_epoch=n_steps,\n                        validation_data=valid_dataset,\n                        epochs=EPOCHS,\n                        verbose=True, \n                        callbacks=[lr_schedule]\n        )\n      \n        print(\"Done training! Now predicting\")\n        oof_predictions.append(model.predict(test_dataset, verbose=1))\n    return oof_predictions\n\noof_predictions = train(x_train,y_train)\navged = sum(oof_predictions)/float(K)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = avged#model.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)\nsub.toxic.hist(bins=100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#n_steps = x_train.shape[0] // BATCH_SIZE\n# train_history = model.fit(\n#                         train_dataset,\n#                         steps_per_epoch=n_steps,\n#                         validation_data=valid_dataset,\n#                         epochs=10\n#                 )\n#--------------------------------------------------------------------------------\n# n_steps = x_valid.shape[0] // BATCH_SIZE  #since generator is used\n# train_history_2 = model.fit(\n#     valid_dataset.repeat(),\n#     steps_per_epoch=n_steps,\n#     epochs=10\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# \"\"\"\n# ------------------------------------------\n# REF: https://github.com/optuna/optuna/blob/master/examples/pruning/tfkeras_integration.py\n# -----------------------------------------\n# Optuna example that demonstrates a pruner for tf.keras.\n# In this example, we optimize the validation accuracy of hand-written digit recognition\n# using tf.keras and MNIST, where the architecture of the neural network\n# and the parameters of optimizer are optimized.\n# Throughout the training of neural networks,\n# a pruner observes intermediate results and stops unpromising trials.\n# \"\"\"\n\n# import tensorflow as tf\n# import tensorflow_datasets as tfds\n\n# import optuna\n# from optuna.integration import TFKerasPruningCallback\n\n\n# BATCHSIZE = 128\n# CLASSES = 10\n# EPOCHS = 20\n# N_TRAIN_EXAMPLES = 3000\n# STEPS_PER_EPOCH = int(N_TRAIN_EXAMPLES / BATCHSIZE / 10)\n# VALIDATION_STEPS = 30\n\n\n# def train_dataset():\n\n#     ds = tfds.load(\"mnist\", split=tfds.Split.TRAIN, shuffle_files=True)\n#     ds = ds.map(lambda x: (tf.cast(x[\"image\"], tf.float32) / 255.0, x[\"label\"]))\n#     ds = ds.repeat().shuffle(1024).batch(BATCHSIZE)\n#     ds = ds.prefetch(tf.data.experimental.AUTOTUNE)\n#     return ds\n\n\n# def eval_dataset():\n\n#     ds = tfds.load(\"mnist\", split=tfds.Split.TEST, shuffle_files=False)\n#     ds = ds.map(lambda x: (tf.cast(x[\"image\"], tf.float32) / 255.0, x[\"label\"]))\n#     ds = ds.repeat().batch(BATCHSIZE)\n#     ds = ds.prefetch(tf.data.experimental.AUTOTUNE)\n#     return ds\n\n\n# def create_model(trial):\n\n#     # Hyperparameters to be tuned by Optuna.\n#     lr = trial.suggest_loguniform(\"lr\", 1e-4, 1e-1)\n#     momentum = trial.suggest_uniform(\"momentum\", 0.0, 1.0)\n#     units = trial.suggest_categorical(\"units\", [32, 64, 128, 256, 512])\n#     '''\n#     lr: 0.09120307278561411\n#     momentum: 0.9124601201218243\n#     units: 256\n#     '''\n#     # Compose neural network with one hidden layer.\n#     model = tf.keras.Sequential()\n#     model.add(tf.keras.layers.Flatten())\n#     model.add(tf.keras.layers.Dense(units=units, activation=tf.nn.relu))\n#     model.add(tf.keras.layers.Dense(CLASSES, activation=tf.nn.softmax))\n\n#     # Compile model.\n#     model.compile(\n#         optimizer=tf.keras.optimizers.SGD(lr=lr, momentum=momentum, nesterov=True),\n#         loss=\"sparse_categorical_crossentropy\",\n#         metrics=[\"accuracy\"],\n#     )\n\n#     return model\n\n\n# def objective(trial):\n#     # Clear clutter from previous TensorFlow graphs.\n#     tf.keras.backend.clear_session()\n\n#     # Metrics to be monitored by Optuna.\n#     if tf.__version__ >= \"2\":\n#         monitor = \"val_accuracy\"\n#     else:\n#         monitor = \"val_acc\"\n\n#     # Create tf.keras model instance.\n#     model = create_model(trial)\n\n#     # Create dataset instance.\n#     ds_train = train_dataset()\n#     ds_eval = eval_dataset()\n\n#     # Create callbacks for early stopping and pruning.\n#     callbacks = [\n#         tf.keras.callbacks.EarlyStopping(patience=3),\n#         TFKerasPruningCallback(trial, monitor),\n#     ]\n\n#     # Train model.\n#     history = model.fit(\n#         ds_train,\n#         epochs=EPOCHS,\n#         steps_per_epoch=STEPS_PER_EPOCH,\n#         validation_data=ds_eval,\n#         validation_steps=VALIDATION_STEPS,\n#         callbacks=callbacks,\n#     )\n\n#     # TODO(@sfujiwara): Investigate why the logger here is called twice.\n#     # tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.DEBUG)\n#     # tf.compat.v1.logging.info('hello optuna')\n\n#     return history.history[monitor][-1]\n\n\n# def show_result(study):\n\n#     pruned_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.PRUNED]\n#     complete_trials = [t for t in study.trials if t.state == optuna.trial.TrialState.COMPLETE]\n\n#     print(\"Study statistics: \")\n#     print(\"  Number of finished trials: \", len(study.trials))\n#     print(\"  Number of pruned trials: \", len(pruned_trials))\n#     print(\"  Number of complete trials: \", len(complete_trials))\n\n#     print(\"Best trial:\")\n#     trial = study.best_trial\n\n#     print(\"  Value: \", trial.value)\n\n#     print(\"  Params: \")\n#     for key, value in trial.params.items():\n#         print(\"    {}: {}\".format(key, value))\n\n\n# def main():\n\n#     study = optuna.create_study(\n#         direction=\"maximize\", pruner=optuna.pruners.MedianPruner(n_startup_trials=2)\n#     )\n\n#     study.optimize(objective, n_trials=25, timeout=600)\n\n#     show_result(study)\n\n\n# if __name__ == \"__main__\":\n#     main()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import json \n# import requests \n# api_key = ''\n# url = ('https://commentanalyzer.googleapis.com/v1alpha1/comments:analyze' + '?key=' + api_key)\n# data_dict = {\n#     'comment': {'text': 'what kind of idiot name is foo?'},\n#     'languages': ['en'],\n#     'requestedAttributes': {'TOXICITY': {}}\n# }\n# response = requests.post(url=url, data=json.dumps(data_dict)) \n# response_dict = json.loads(response.content) \n# print(json.dumps(response_dict, indent=2))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}