{"cells":[{"metadata":{"id":"kNr-CtKM8jT3","trusted":true},"cell_type":"code","source":"from pathlib import Path\nimport numpy as np \nimport pandas as pd \nfrom sklearn.utils import shuffle\nfrom sklearn.metrics import roc_auc_score\nimport tensorflow as tf\nfrom tqdm import tqdm\n\nimport transformers\n\nfrom transformers import PretrainedConfig, TFRobertaModel\n\nfrom tokenizers import BertWordPieceTokenizer, SentencePieceBPETokenizer\n\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import regularizers\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nimport matplotlib.pyplot as plt\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"id":"tFAM9gZl8jT_","outputId":"1b2f0498-ea8a-4327-c9e1-a83d1f301600","trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_ROOT = Path(\"..\")/\"input\"/ \"jigsaw-multilingual-toxic-comment-classification/\"\n\ndf1,df2,df3,test,sample = [pd.read_csv(DATA_ROOT/fname) for fname in [\"jigsaw-toxic-comment-train.csv\",\n                                                                        \"jigsaw-unintended-bias-train.csv\",\n                                                                        \"validation.csv\",\n                                                                        \"test.csv\",\n                                                                        \"sample_submission.csv\"\n                                                                       ]]\n\ntrain = df1\nvalid = df3\n\n# subsample the train dataframe to 50%-50% \ntrain = pd.concat([\n    train.query('toxic==1'),\n    train.query('toxic==0').sample(117700,random_state=42)\n])\n# shufle it just to make sure \ntrain = train.sample(frac=1, random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sum(train.toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"id":"OgE2MNL28jUD","trusted":true},"cell_type":"code","source":"# #DATA_ROOT = Path(\"..\")/\"input\"/ \"jigsaw-multilingual-toxic-comment-classification/\"\n\n# df1,df2,df3,test,sample = [pd.read_csv(fname) for fname in [\"jigsaw-toxic-comment-train.csv\",\n#                                                                         \"jigsaw-unintended-bias-train.csv\",\n#                                                                         \"validation.csv\",\n#                                                                         \"test.csv\",\n#                                                                         \"sample_submission.csv\"\n#                                                                        ]]\n# df2.toxic = df2.toxic.round().astype(int)\n# train = pd.concat([\n#     df1[['comment_text', 'toxic']],\n#     df2[['comment_text', 'toxic']].query('toxic==1'),\n#     df2[['comment_text', 'toxic']].query('toxic==0').sample(n=100000, random_state=0)\n# ])\n\n# valid = df3\n\n\n# # train = pd.read_csv('jigsaw-toxic-comment-train.csv')\n# # valid = pd.read_csv('validation.csv')\n# # test = pd.read_csv('test.csv')\n# # submission = pd.read_csv('sample_submission.csv')\n\n# # subsample the train dataframe to 50%-50% \n# # train = pd.concat([\n# #     train.query('toxic==1'),\n# #     train.query('toxic==0').sample(sum(train.toxic),random_state=42)\n# # ])\n# # shufle it just to make sure \n# train = train.sample(frac=1, random_state = 42)\n\n\n## Getting data in new way to improve\n","execution_count":null,"outputs":[]},{"metadata":{"id":"c9oMno1ODShZ","trusted":true},"cell_type":"code","source":"# df1,df2,df3 = [pd.read_csv(fname) for fname in [\"jigsaw-toxic-comment-train.csv\",\n#                                                 \"jigsaw-unintended-bias-train.csv\",\n#                                                 \"validation.csv\"\n#                                               ]]\n\n\n\n# test, sample= [pd.read_csv(fname) for fname in [\"test.csv\",\n#                                                 \"sample_submission.csv\"\n#                                                ]]","execution_count":null,"outputs":[]},{"metadata":{"id":"NqRd_hfBEe3I","trusted":true},"cell_type":"code","source":"# df2['toxic'] = df2['toxic'].apply(lambda x: 0 if x<0.5 else 1)","execution_count":null,"outputs":[]},{"metadata":{"id":"SYtdytKZEjfD","trusted":true},"cell_type":"code","source":"# df2 = pd.concat([\n    \n#     df2[['comment_text', 'toxic']].query('toxic==1'),\n#     df2[['comment_text', 'toxic']].query('toxic==0').sample(n=500000, random_state=0)\n# ])","execution_count":null,"outputs":[]},{"metadata":{"id":"cFOeen34EkOZ","outputId":"88ae9b54-2d35-4602-d9cd-11d7b7663b18","trusted":true},"cell_type":"code","source":"# df2.toxic.value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"wXSfLfV1E8Ty","trusted":true},"cell_type":"code","source":"# # Character length for the rows in the df1 & df2 data\n# df1['char_length'] = df1['comment_text'].apply(lambda x: len(str(x)))\n# df2['char_length'] = df2['comment_text'].apply(lambda x: len(str(x)))","execution_count":null,"outputs":[]},{"metadata":{"id":"HoeYEKMDE9c6","trusted":true},"cell_type":"code","source":"# # Character length for the rows in the training data\n# df1= df1[df1['char_length'] >= 545] \n# df2= df2[df2['char_length'] >= 545] ","execution_count":null,"outputs":[]},{"metadata":{"id":"ooEuaAdMFASY","trusted":true},"cell_type":"code","source":"# train = pd.concat([df1, df2], axis=0).reset_index(drop=True)\n# train = train.sample(frac=1).reset_index(drop=True).head(200000)\n# train = train.reset_index(drop=True)\n# valid = df3\n# valid = valid.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"JitD7blp8jUH","outputId":"869e6bbc-1517-4c92-b12b-31f6647111ec","trusted":true},"cell_type":"code","source":"# Check percentage of toxic comments in training Dataset\ntrain.toxic.value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"DBI0glH58jUN","outputId":"0db68f52-4d09-475d-cdad-a1915f61eb3e","trusted":true},"cell_type":"code","source":"# Check percentage of toxic comments in validation Dataset\nvalid.toxic.value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"id":"b2nmxvOZ8jUS","trusted":true},"cell_type":"code","source":"# # # Function to encode the text\ndef encode_fn(texts, tokenizer, maxlen=512):\n    encode = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(encode['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"id":"o0PtW5O-8jUW","trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n# Configuration\n#EPOCHS = 3\nLR = 1e-5\nBATCH_SIZE = 8 \nglobal_BATCH_SIZE = BATCH_SIZE * strategy.num_replicas_in_sync\nMAX_LEN = 192\nTOTAL_STEPS_STAGE1 = 3000\nVALIDATE_EVERY_STAGE1 = 500\nTOTAL_STEPS_STAGE2 = 1500\nVALIDATE_EVERY_STAGE2 = 500\n\n# roberta-base\n#CONFIG_PATH = '../input/robertabaseconfig/config.json'\n# roberta-large\nCONFIG_PATH = '../input/robertalargeconfig/config.json'","execution_count":null,"outputs":[]},{"metadata":{"id":"jYJswGhB8jUc","trusted":true},"cell_type":"code","source":"# First load the real tokenizer\n#tokenizer = transformers.AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-base')\n# First load the real tokenizer\ntokenizer = transformers.AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large')","execution_count":null,"outputs":[]},{"metadata":{"id":"3xxsqhWL8jUl","outputId":"ce340d2c-f5ae-4ebf-87db-005140042b87","trusted":true},"cell_type":"code","source":"%%time\nx_train = encode_fn(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)\nx_valid = encode_fn(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)\nx_test =  encode_fn(test.content.astype(str), tokenizer, maxlen=MAX_LEN)\n\ny_train = train.toxic.values.reshape(-1,1)\ny_valid = valid.toxic.values.reshape(-1,1)","execution_count":null,"outputs":[]},{"metadata":{"id":"4CcfV0qC0B-0","trusted":true},"cell_type":"code","source":"def create_dataset(X, y=None, training=False):\n    dataset = tf.data.Dataset.from_tensor_slices(X)\n\n    ### Add y if present ###\n    if y is not None:\n        dataset_y = tf.data.Dataset.from_tensor_slices(y)\n        dataset = tf.data.Dataset.zip((dataset, dataset_y))\n        \n    ### Repeat if training ###\n    if training:\n        dataset = dataset.shuffle(len(X)).repeat()\n\n    dataset = dataset.batch(global_BATCH_SIZE).prefetch(AUTO)\n\n    ### make it distributed  ###\n    dist_dataset = strategy.experimental_distribute_dataset(dataset)\n\n    return dist_dataset\n","execution_count":null,"outputs":[]},{"metadata":{"id":"IrzeLGce0OAJ","trusted":true},"cell_type":"code","source":"# Create datasets\ntrain_dataset = create_dataset(x_train, y_train, True)\nvalid_dataset   = create_dataset(x_valid)\ntest_dataset  = create_dataset(x_test)","execution_count":null,"outputs":[]},{"metadata":{"id":"OEFMC8Df8jU8","trusted":true},"cell_type":"code","source":"# Function to build the MODEL\ndef model_fn(transformer, max_len=512):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    \n    cls_token = sequence_output[:, 0, :]\n    \n    \n    # layers1 = Dense(512, kernel_regularizer=regularizers.l2(hp.get('regularize')),activation='elu')(cls_token)\n    # #layers2 = Dropout(0.5)(layers1)\n    # layers3 = Dense(512, kernel_regularizer=regularizers.l2(hp.get('regularize')),activation='elu')(layers1)\n    # #layes4 = Dropout(0.5)(layers3)\n    # layers5 = Dense(512, kernel_regularizer=regularizers.l2(hp.get('regularize')),activation='elu')(layers3)\n    # #layers6 = Dropout(0.5)(layers5)\n    # layers7 = Dense(512, kernel_regularizer=regularizers.l2(hp.get('regularize')),activation='elu')(layers5)\n    # #layers8 = Dropout(0.5)(layers7)\n    \n    \n    \n    #out = Dense(1, activation='sigmoid')(layers7)\n    out = Dense(1, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    #model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics = [tf.keras.metrics.AUC()] )#metrics=['accuracy'])\n    #model.compile(Adam(lr=hp.get('learning_rate')), loss='binary_crossentropy',metrics=['accuracy']) #metrics = [tf.keras.metrics.AUC()] )#metrics=['accuracy'])\n    \n    \n    \n    return model\n    \n    \n    \n#     cls_token = sequence_output[:, 0, :]\n#     out = Dense(1, activation='sigmoid')(cls_token)\n    \n#     model = Model(inputs=input_word_ids, outputs=out)\n#     model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])\n#     model.compile(\n#         loss=tf.keras.losses.BinaryCrossentropy(),\n#         optimizer=tf.keras.optimizers.SGD(learning_rate=0.001),\n#         metrics=[tf.keras.metrics.AUC()]\n    \n#    )\n    ","execution_count":null,"outputs":[]},{"metadata":{"id":"0wwk35TCl9a4","outputId":"04aa0ee4-c661-4b23-8bfc-f54fbd946fc6","trusted":true},"cell_type":"code","source":"# ## Download config of huggingface roberta base model\n# #!wget https://s3.amazonaws.com/models.huggingface.co/bert/jplu/tf-xlm-roberta-large/config.json\n# !wget https://s3.amazonaws.com/models.huggingface.co/bert/jplu/tf-xlm-roberta-base/config.json","execution_count":null,"outputs":[]},{"metadata":{"id":"WHP0_3P4lMc0","outputId":"8f73d1d3-70dc-4746-f785-9d8a5ec64c07","trusted":true},"cell_type":"code","source":"%%time\ndef create_model_from_config():\n    with strategy.scope():\n        ### Load only config no weights ###\n        config = PretrainedConfig.from_json_file(CONFIG_PATH)                \n        transformer_layer = TFRobertaModel(config) \n\n        ### Make the cls model ###               \n        model = model_fn(transformer_layer)\n        optimizer = tf.keras.optimizers.Adam(learning_rate=LR)\n    model.summary()\n    return model, optimizer\n\n\ndef load_weights_workaround():\n    ### Load full pretrained model outside strategy scope ###\n    #transformer_layer = transformers.TFAutoModel.from_pretrained('jplu/tf-xlm-roberta-base')\n    transformer_layer = transformers.TFAutoModel.from_pretrained('jplu/tf-xlm-roberta-large')\n\n    ### Assign weights \n    for tv1, tv2 in zip(model.layers[1].trainable_variables,\n                        transformer_layer.trainable_variables):\n        tv1.assign(tv2)\n\n\nmodel, optimizer = create_model_from_config()\nload_weights_workaround()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"id":"bKpsoq9soedX","trusted":true},"cell_type":"code","source":"def train(train_dataset, valid_dataset=None, y_valid=None,\n          total_steps=5000, validate_every=500):\n    step = 0\n    ### Training lopp ###\n    for tensor in train_dataset:\n        distributed_train_step(tensor) \n        step+=1\n\n        if (step % validate_every == 0):   \n            ### Print train metrics ###  \n            train_metric = train_accuracy_metric.result().numpy()\n            print(\"Step %d, train AUC: %.5f\" % (step, train_metric))   \n            \n            ### Test loop with exact AUC ###\n            if valid_dataset:\n                val_metric = roc_auc_score(y_valid, predict(valid_dataset))\n                print(\"     validation AUC: %.5f\" %  val_metric)   \n\n            ### Reset (train) metrics ###\n            train_accuracy_metric.reset_states()\n            \n        if step  == total_steps:\n            break","execution_count":null,"outputs":[]},{"metadata":{"id":"6z7lYFrCpZoM","trusted":true},"cell_type":"code","source":"@tf.function\ndef distributed_train_step(data):\n    strategy.experimental_run_v2(train_step, args=(data,))","execution_count":null,"outputs":[]},{"metadata":{"id":"XQF570vvpaQL","trusted":true},"cell_type":"code","source":"# def train_step(inputs):\n#     features, labels = inputs\n\n#     with tf.GradientTape() as tape:\n#         predictions = model(features, training=True)\n#         loss = compute_loss(labels, predictions)\n\n#     gradients = tape.gradient(loss, model.trainable_variables)\n#     optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n\n#     train_accuracy_metric.update_state(labels, predictions)","execution_count":null,"outputs":[]},{"metadata":{"id":"ASzQFUb74dIV","trusted":true},"cell_type":"code","source":"def predict(dataset):  \n    predictions = []\n    for tensor in dataset:\n        predictions.append(distributed_prediction_step(tensor))\n    ### stack replicas and batches\n    predictions = np.vstack(list(map(np.vstack,predictions)))\n    return predictions\n\n@tf.function\ndef distributed_prediction_step(data):\n    predictions = strategy.experimental_run_v2(prediction_step, args=(data,))\n    return strategy.experimental_local_results(predictions)\n\ndef prediction_step(inputs):\n    features = inputs  # note datasets used in prediction do not have labels\n    predictions = model(features, training=False)\n    return predictions","execution_count":null,"outputs":[]},{"metadata":{"id":"LUIXXKajp5C_","trusted":true},"cell_type":"code","source":"def define_losses_and_metrics():\n    with strategy.scope():\n        loss_object = tf.keras.losses.BinaryCrossentropy(\n            reduction=tf.keras.losses.Reduction.NONE, from_logits=False)\n\n        def compute_loss(labels, predictions):\n            per_example_loss = loss_object(labels, predictions)\n            loss = tf.nn.compute_average_loss(\n                per_example_loss, global_batch_size = global_BATCH_SIZE)\n            return loss\n\n        train_accuracy_metric = tf.keras.metrics.AUC(name='training_AUC')\n\n    return compute_loss, train_accuracy_metric","execution_count":null,"outputs":[]},{"metadata":{"id":"U2pGARuOqOKb","trusted":true},"cell_type":"code","source":"compute_loss, train_accuracy_metric = define_losses_and_metrics()","execution_count":null,"outputs":[]},{"metadata":{"id":"pqkJhJZIofxX","trusted":true},"cell_type":"code","source":"# #%%time\n# train(train_dataset, valid_dataset, y_valid,\n#       TOTAL_STEPS_STAGE1, VALIDATE_EVERY_STAGE1)\n\n\n# ResourceExhaustedError will occur","execution_count":null,"outputs":[]},{"metadata":{"id":"RQS88bT5oldo","trusted":true},"cell_type":"code","source":"with strategy.scope():\n    optimizer = tf.keras.optimizers.SGD(learning_rate=LR)","execution_count":null,"outputs":[]},{"metadata":{"id":"dpDAGcJkwwU_","trusted":true},"cell_type":"code","source":"CLIP_NORM = 1  # agressive clipping\n\n@tf.function\ndef train_step(data):\n    inputs, targets = data\n    with tf.GradientTape() as tape:\n        predictions = model(inputs, training=True)\n        loss = compute_loss(targets, predictions)\n\n    ### There is an unused pooler head of the tranformer with None gradients\n    ### we need to get rid of it before clipping\n    trainable_variables = [v for v in model.trainable_variables \n                           if 'pooler' not in v.name]\n\n    ### Calculate grads\n    gradients = tape.gradient(loss, trainable_variables)\n    \n    ### We cannot clip replicas, it throws an error\n    ### First we have to manually sum the gradients from the replicas\n    gradients = tf.distribute.get_replica_context().all_reduce('sum', gradients)\n\n    ### Clip by global norm, (do not change gradient direction)\n    gradients, _ = tf.clip_by_global_norm(gradients, CLIP_NORM)\n\n    ### Apply gradients\n    ### NOTE: Only for tenforflow 2.2 on colab!!!!\n    optimizer.apply_gradients(zip(gradients, trainable_variables))\n    \n    #                          experimental_aggregate_gradients=False)\n\n    train_accuracy_metric.update_state(targets, predictions)","execution_count":null,"outputs":[]},{"metadata":{"id":"p6uLFWFWw0YG","outputId":"48824ec4-a17f-47bb-bc9a-6cc750b4b4cf","trusted":true},"cell_type":"code","source":"optimizer.learning_rate.assign(0.001)","execution_count":null,"outputs":[]},{"metadata":{"id":"kvEl9Bb9w1pN","outputId":"2ff23919-1786-4348-a445-9f48307df77e","trusted":true},"cell_type":"code","source":"#%%time\ntrain(train_dataset, valid_dataset, y_valid,\n      TOTAL_STEPS_STAGE1, VALIDATE_EVERY_STAGE1)\n","execution_count":null,"outputs":[]},{"metadata":{"id":"0pTdD2wcBCOX","outputId":"84fff6c1-1c33-43e1-ea5d-d557c85b3919","trusted":true},"cell_type":"code","source":"%%time\n# make a new dataset for training with the validation data \n# with targets, shuffling and repeating\nvalid_dataset_4_training = create_dataset(x_valid, y_valid, training=True)\n\n# train again\ntrain(valid_dataset_4_training,\n      total_steps = TOTAL_STEPS_STAGE2, \n      validate_every = VALIDATE_EVERY_STAGE2)  # not validating but printing now","execution_count":null,"outputs":[]},{"metadata":{"id":"IC1eobWt8jVa","outputId":"16d5673e-ad48-4182-b0cb-7759ecf4cfb4","trusted":true},"cell_type":"code","source":"sample['toxic'] = predict(test_dataset)\nsample.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}