{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Importing the files and installing stuffs","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q gcsfs","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, time\nimport pandas\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom kaggle_datasets import  KaggleDatasets \nimport warnings\nwarnings.filterwarnings('ignore')\n\n\n# We'll use a tokenizer for the BERT model from the modelling demo notebook.\n!pip install bert-tensorflow\nimport bert.tokenization\n\nprint(tf.version.VERSION)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#select tpu\n# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set data paths\n\nSet maximum sequence length and path variables.","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"SEQUENCE_LENGTH = 128\n\n# Note that private datasets cannot be copied - you'll have to share any pretrained models \n# you want to use with other competitors!\nGCS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\nBERT_GCS_PATH = KaggleDatasets().get_gcs_path('bert-multi')\nBERT_GCS_PATH_SAVEDMODEL = BERT_GCS_PATH + \"/bert_multi_from_tfhub\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Multi lingual BERT","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def multilingual_bert_model(max_seq_length=SEQUENCE_LENGTH, trainable_bert=True):\n    \"\"\"Build and return a multilingual BERT model and tokenizer.\"\"\"\n    input_word_ids = tf.keras.layers.Input(\n        shape=(max_seq_length,), dtype=tf.int32, name=\"input_word_ids\")\n    input_mask = tf.keras.layers.Input(\n        shape=(max_seq_length,), dtype=tf.int32, name=\"input_mask\")\n    segment_ids = tf.keras.layers.Input(\n        shape=(max_seq_length,), dtype=tf.int32, name=\"all_segment_id\")\n    \n    # Load a SavedModel on TPU from GCS. This model is available online at \n    # https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/1. You can use your own \n    # pretrained models, but will need to add them as a Kaggle dataset.\n    bert_layer = tf.saved_model.load(BERT_GCS_PATH_SAVEDMODEL)\n    # Cast the loaded model to a TFHub KerasLayer.\n    bert_layer = hub.KerasLayer(bert_layer, trainable=trainable_bert)\n\n    pooled_output, _ = bert_layer([input_word_ids, input_mask, segment_ids])\n    output = tf.keras.layers.Dense(32)(pooled_output)\n    output = tf.keras.layers.LeakyReLU()(output)\n    output = tf.keras.layers.Dense(1, activation='sigmoid', name='labels')(output)\n\n    return tf.keras.Model(inputs={'input_word_ids': input_word_ids,\n                                  'input_mask': input_mask,\n                                  'all_segment_id': segment_ids},\n                          outputs=output)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"At this point they have looked into examples of their previous competition, but I will see the datas of this competition\n\n# train, test, validation files","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training data from our first competition,\n# https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge/data\ntrain_name = GCS_PATH + \"/jigsaw-toxic-comment-train-processed-seqlen128.csv\"\ntest_name = GCS_PATH + \"/test.csv\"\nvalidation_name = GCS_PATH + \"/validation.csv\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pandas.read_csv(train_name)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pandas.read_csv(test_name, encoding= 'utf-8')\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation = pandas.read_csv(validation_name, encoding = 'utf')\nvalidation.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BERT Tokenizer\n\nGet the tokenizer corresponding to our multilingual BERT model. See [TensorFlow \nHub](https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/1) for more information about the model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tokenizer(bert_path=BERT_GCS_PATH_SAVEDMODEL):\n    \"\"\"Get the tokenizer for a BERT layer.\"\"\"\n    bert_layer = tf.saved_model.load(bert_path)\n    bert_layer = hub.KerasLayer(bert_layer, trainable = False)\n    vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n    cased = bert_layer.resolved_object.do_lower_case.numpy()\n    tf.gfile = tf.io.gfile  # for bert.tokenization.load_vocab in tokenizer\n    tokenizer = bert.tokenization.FullTokenizer(vocab_file, cased)\n  \n    return tokenizer\n\ntokenizer = get_tokenizer()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can look at one of our example sentences after we tokenize it, and then again after converting it to word IDs for BERT.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"example_sentence = train.iloc[37].comment_text[:150]\nprint(example_sentence)\n\nexample_tokens = tokenizer.tokenize(example_sentence)\nprint(example_tokens[:17])\n\nexample_input_ids = tokenizer.convert_tokens_to_ids(example_tokens)\nprint(example_input_ids[:17])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing\n\nProcess individual sentences for input to BERT using the tokenizer, and then prepare the entire dataset. The same code will process the other training data files, as well as the validation and test data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_sentence(sentence, max_seq_length=SEQUENCE_LENGTH, tokenizer=tokenizer):\n    \"\"\"Helper function to prepare data for BERT. Converts sentence input examples\n    into the form ['input_word_ids', 'input_mask', 'segment_ids'].\"\"\"\n    # Tokenize, and truncate to max_seq_length if necessary.\n    tokens = tokenizer.tokenize(sentence)\n    if len(tokens) > max_seq_length - 2:\n        tokens = tokens[:(max_seq_length - 2)]\n\n    # Convert the tokens in the sentence to word IDs.\n    input_ids = tokenizer.convert_tokens_to_ids([\"[CLS]\"] + tokens + [\"[SEP]\"])\n\n    # The mask has 1 for real tokens and 0 for padding tokens. Only real\n    # tokens are attended to.\n    input_mask = [1] * len(input_ids)\n\n    # Zero-pad up to the sequence length.\n    pad_length = max_seq_length - len(input_ids)\n    input_ids.extend([0] * pad_length)\n    input_mask.extend([0] * pad_length)\n\n    # We only have one input segment.\n    segment_ids = [0] * max_seq_length\n\n    return (input_ids, input_mask, segment_ids)\n\ndef preprocess_and_save_dataset(unprocessed_filename, text_label='comment_text',\n                                seq_length=SEQUENCE_LENGTH, verbose=True):\n    \"\"\"Preprocess a CSV to the expected TF Dataset form for multilingual BERT,\n    and save the result.\"\"\"\n    dataframe = pandas.read_csv(unprocessed_filename,\n                                index_col='id', encoding = 'utf-8')\n    processed_filename = (\"unprocessed_filename.rstrip('.csv')\" +\n                          \"-processed-seqlen{}.csv\".format(SEQUENCE_LENGTH))\n\n    pos = 0\n    start = time.time()\n\n    while pos < len(dataframe):\n        processed_df = dataframe[pos:pos + 10000].copy() \n        processed_df['input_word_ids'], processed_df['input_mask'], processed_df['all_segment_id'] = (\n            zip(*processed_df[text_label].apply(process_sentence)))\n\n        if pos == 0:\n            processed_df.to_csv(processed_filename, index_label='id', mode='w')\n        else:\n             processed_df.to_csv(processed_filename, index_label='id', mode='a',\n                                header=False)\n\n        if verbose:\n            print('Processed {} examples in {}'.format(\n                pos + 10000, time.time() - start))\n        pos += 10000\n    return\n  \n# Process the training dataset.\npreprocess_and_save_dataset(train_name, text_label='comment_text',\n                                seq_length=SEQUENCE_LENGTH, verbose=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def parse_string_list_into_ints(strlist):\n    s = tf.strings.strip(strlist)\n    s = tf.strings.substr(\n        strlist, 1, tf.strings.length(s) - 2)  # Remove parentheses around list\n    s = tf.strings.split(s, ',', maxsplit=SEQUENCE_LENGTH)\n    s = tf.strings.to_number(s, tf.int32)\n    s = tf.reshape(s, [SEQUENCE_LENGTH])  # Force shape here needed for XLA compilation (TPU)\n    return s\n\ndef format_sentences(data, label='toxic', remove_language=False):\n    labels = {'labels': data.pop(label)}\n    if remove_language:\n        languages = {'language': data.pop('lang')}\n    # The remaining three items in the dict parsed from the CSV are lists of integers\n    for k,v in data.items():  # \"input_word_ids\", \"input_mask\", \"all_segment_id\"\n        data[k] = parse_string_list_into_ints(v)\n    return data, labels\n\ndef make_sentence_dataset_from_csv(filename, label='toxic', language_to_filter=None):\n    # This assumes the column order label, input_word_ids, input_mask, segment_ids\n    SELECTED_COLUMNS = [label, \"input_word_ids\", \"input_mask\", \"all_segment_id\"]\n    label_default = tf.int32 if label == 'id' else tf.float32\n    COLUMN_DEFAULTS  = [label_default, tf.string, tf.string, tf.string]\n\n    if language_to_filter:\n        insert_pos = 0 if label != 'id' else 1\n        SELECTED_COLUMNS.insert(insert_pos, 'lang')\n        COLUMN_DEFAULTS.insert(insert_pos, tf.string)\n\n    preprocessed_sentences_dataset = tf.data.experimental.make_csv_dataset(\n        filename, column_defaults=COLUMN_DEFAULTS, select_columns=SELECTED_COLUMNS,\n        batch_size=1, num_epochs=1, shuffle=False)  # We'll do repeating and shuffling ourselves\n    # make_csv_dataset required a batch size, but we want to batch later\n    preprocessed_sentences_dataset = preprocessed_sentences_dataset.unbatch()\n    \n    if language_to_filter:\n        preprocessed_sentences_dataset = preprocessed_sentences_dataset.filter(\n            lambda data: tf.math.equal(data['lang'], tf.constant(language_to_filter)))\n        #preprocessed_sentences.pop('lang')\n    preprocessed_sentences_dataset = preprocessed_sentences_dataset.map(\n        lambda data: format_sentences(data, label=label,\n                                      remove_language=language_to_filter))\n\n    return preprocessed_sentences_dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Setting up our data pipelines for training and evaluation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def make_dataset_pipeline(dataset, repeat_and_shuffle=True):\n    \"\"\"Set up the pipeline for the given dataset.\n    \n    Caches, repeats, shuffles, and sets the pipeline up to prefetch batches.\"\"\"\n    cached_dataset = dataset.cache()\n    if repeat_and_shuffle:\n        cached_dataset = cached_dataset.repeat().shuffle(2048)\n    cached_dataset = cached_dataset.batch(32 * strategy.num_replicas_in_sync)\n    cached_dataset = cached_dataset.prefetch(tf.data.experimental.AUTOTUNE)\n    return cached_dataset\n\n# Load the preprocessed English dataframe.\npreprocessed_en_filename = (\n    GCS_PATH + \"/jigsaw-toxic-comment-train-processed-seqlen{}.csv\".format(\n        SEQUENCE_LENGTH))\n\n# Set up the dataset and pipeline.\nenglish_train_dataset = make_dataset_pipeline(\n    make_sentence_dataset_from_csv(preprocessed_en_filename))\n\n# Process the new datasets by language.\npreprocessed_val_filename = (\n    GCS_PATH + \"/validation-processed-seqlen{}.csv\".format(SEQUENCE_LENGTH))\n\nnonenglish_val_datasets = {}\nfor language_name, language_label in [('Spanish', 'es'), ('Italian', 'it'),\n                                      ('Turkish', 'tr')]:\n    nonenglish_val_datasets[language_name] = make_sentence_dataset_from_csv(\n        preprocessed_val_filename, language_to_filter=language_label)\n    nonenglish_val_datasets[language_name] = make_dataset_pipeline(\n        nonenglish_val_datasets[language_name])\n\nnonenglish_val_datasets['Combined'] = tf.data.experimental.sample_from_datasets(\n        (nonenglish_val_datasets['Spanish'], nonenglish_val_datasets['Italian'],\n         nonenglish_val_datasets['Turkish']))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Compile our model. We'll first evaluate it on our new toxicity dataset in the different languages to see its performance. After that, we'll train it on one of our English datasets, and then again evaluate its performance on the new multilingual toxicity data. As our metric, we'll use the AUC.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    multilingual_bert = multilingual_bert_model()\n\n    # Compile the model. Optimize using stochastic gradient descent.\n    multilingual_bert.compile(\n        loss=tf.keras.losses.BinaryCrossentropy(),\n        optimizer=tf.keras.optimizers.SGD(learning_rate=0.001),\n        metrics=[tf.keras.metrics.AUC()])\n\nmultilingual_bert.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Test the model's performance on non-English comments before training.\nfor language in nonenglish_val_datasets:\n    results = multilingual_bert.evaluate(nonenglish_val_datasets[language],\n                                         steps=100, verbose=0)\n    print('{} loss, AUC before training:'.format(language), results)\n\nresults = multilingual_bert.evaluate(english_train_dataset,\n                                     steps=100, verbose=0)\nprint('\\nEnglish loss, AUC before training:', results)\n\nprint()\n# Train on English Wikipedia comment data.\nhistory = multilingual_bert.fit(\n    # Set steps such that the number of examples per epoch is fixed.\n    # This makes training on different accelerators more comparable.\n    english_train_dataset, steps_per_epoch=4000/strategy.num_replicas_in_sync,\n    epochs=100, verbose=1, validation_data=nonenglish_val_datasets['Combined'],\n    validation_steps=500)\nprint()\n\n# Re-evaluate the model's performance on non-English comments after training.\nfor language in nonenglish_val_datasets:\n    results = multilingual_bert.evaluate(nonenglish_val_datasets[language],\n                                         steps=100, verbose=0)\n    print('{} loss, AUC after training:'.format(language), results)\n\nresults = multilingual_bert.evaluate(english_train_dataset,\n                                     steps=100, verbose=0)\nprint('\\nEnglish loss, AUC after training:', results)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generate predictions\nFinally, we'll use our trained multilingual model to generate predictions for the test data.****","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np\n\nTEST_DATASET_SIZE = 63812\n\nprint('Making dataset...')\npreprocessed_test_filename = (\n    GCS_PATH + \"/test-processed-seqlen{}.csv\".format(SEQUENCE_LENGTH))\ntest_dataset = make_sentence_dataset_from_csv(preprocessed_test_filename, label='id')\ntest_dataset = make_dataset_pipeline(test_dataset, repeat_and_shuffle=False) \n    \nprint('Computing predictions...')\ntest_sentences_dataset = test_dataset.map(lambda sentence, idnum: sentence)\nprobabilities = np.squeeze(multilingual_bert.predict(test_sentences_dataset))\nprint(probabilities)\n\nprint('Generating submission file...')\ntest_ids_dataset = test_dataset.map(lambda sentence, idnum: idnum).unbatch()\ntest_ids = next(iter(test_ids_dataset.batch(TEST_DATASET_SIZE)))[\n    'labels'].numpy().astype('U')  # All in one batch\n\nnp.savetxt('submission.csv', np.rec.fromarrays([test_ids, probabilities]),\n           fmt=['%s', '%f'], delimiter=',', header='id,toxic', comments='')\n!head submission.csv\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}