{"cells":[{"metadata":{"_cell_guid":"593619be-8090-4dad-a462-e883e560ec1c","_uuid":"cba509cb-708d-4fd0-8e2a-e5c1c7a0982d","trusted":true},"cell_type":"code","source":"import os, time\nimport pandas\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom kaggle_datasets import KaggleDatasets # comment this if not running on Kaggle\nprint(tf.version.VERSION)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"f5ccaf08-c532-4fde-9306-b897c890d0f8","_uuid":"bc97f110-17eb-44dd-a792-d66c27a0b3a6","trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n    print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nexcept ValueError:\n    tpu = None\n    gpus = tf.config.experimental.list_logical_devices(\"GPU\")\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelif len(gpus) > 1: # multiple GPUs in one VM\n    strategy = tf.distribute.MirroredStrategy(gpus)\nelse: # default strategy that works on CPU and single GPU\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n# mixed precision\n# On TPU, bfloat16/float32 mixed precision is automatically used in TPU computations.\n# Enabling it in Keras also stores relevant variables in bfloat16 format (memory optimization).\n# On GPU, specifically V100, mixed precision must be enabled for hardware TensorCores to be used.\n# XLA compilation must be enabled for this to work. (On TPU, XLA compilation is the default)\nMIXED_PRECISION = True\nif MIXED_PRECISION:\n    if tpu: \n        policy = tf.keras.mixed_precision.experimental.Policy('mixed_bfloat16')\n    else: #\n        policy = tf.keras.mixed_precision.experimental.Policy('mixed_float16')\n        tf.config.optimizer.set_jit(True) # XLA compilation\n    tf.keras.mixed_precision.experimental.set_policy(policy)\n    print('Mixed precision enabled')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7acc953d-b1f6-4d29-a006-69af6839f7a9","_uuid":"7d8d39c7-1fd2-4af7-a88a-9ca1242a6277","trusted":true},"cell_type":"code","source":"SEQUENCE_LENGTH = 128\n\n# Note that private datasets cannot be copied - you'll have to share any pretrained models \n# you want to use with other competitors!\nBERT_GCS_PATH = KaggleDatasets().get_gcs_path('bert-multi')\n# BERT_GCS_PATH = gs:// ... # if using your own bucket\n\nBERT_GCS_PATH_SAVEDMODEL = BERT_GCS_PATH + \"/bert_multi_from_tfhub\"\n\nGCS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\n# GCS_PATH = gs:// ... # if using your own bucket\n\nBATCH_SIZE = 64 * strategy.num_replicas_in_sync\n\nTRAIN_DATA = GCS_PATH + \"/jigsaw-toxic-comment-train-processed-seqlen{}.csv\".format(SEQUENCE_LENGTH)\nTRAIN_DATA_LENGTH = 223549 # rows\nVALID_DATA = GCS_PATH + \"/validation-processed-seqlen{}.csv\".format(SEQUENCE_LENGTH)\nSTEPS_PER_EPOCH = TRAIN_DATA_LENGTH // BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Here I changed pretrained model to huggingface transformer."},{"metadata":{"_cell_guid":"fcd2093b-0774-4adf-9e4c-e9096f156d32","_uuid":"1392a5e0-c8e4-46ea-b45d-0d9289682e09","trusted":true},"cell_type":"code","source":"from transformers import TFAutoModel\n\n\ndef multilingual_bert_model(max_seq_length=SEQUENCE_LENGTH, trainable_bert=True):\n    \"\"\"Build and return a multilingual BERT model and tokenizer.\"\"\"\n    input_word_ids = tf.keras.layers.Input(\n        shape=(max_seq_length,), dtype=tf.int32, name=\"input_word_ids\")\n    input_mask = tf.keras.layers.Input(\n        shape=(max_seq_length,), dtype=tf.int32, name=\"input_mask\")\n    segment_ids = tf.keras.layers.Input(\n        shape=(max_seq_length,), dtype=tf.int32, name=\"all_segment_id\")\n    \n    # Load a SavedModel on TPU from GCS. This model is available online at \n    # https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/1. You can use your own \n    # pretrained models, but will need to add them as a Kaggle dataset.\n    \n    #bert_layer = tf.saved_model.load(BERT_GCS_PATH_SAVEDMODEL)\n    # Cast the loaded model to a TFHub KerasLayer.\n    #bert_layer = hub.KerasLayer(bert_layer, trainable=trainable_bert)\n    bert_layer = TFAutoModel.from_pretrained('bert-base-multilingual-uncased')\n    \n    pooled_output, _ = bert_layer([input_word_ids, input_mask, segment_ids])\n    output = tf.keras.layers.Dense(32, activation='relu')(pooled_output)\n    output = tf.keras.layers.Dense(1, activation='sigmoid', name='labels', dtype=tf.float32)(output)\n\n    return tf.keras.Model(inputs={'input_word_ids': input_word_ids,\n                                  'input_mask': input_mask,\n                                  'all_segment_id': segment_ids},\n                          outputs=output)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"422a984e-e571-4898-9667-b95d38416ddd","_uuid":"e3d569ca-0bf4-4bde-aa69-95a46908f65a","trusted":true},"cell_type":"code","source":"with strategy.scope():\n    multilingual_bert = multilingual_bert_model()\n\n    # Compile the model. Optimize using stochastic gradient descent.\n    multilingual_bert.compile(\n        loss=tf.keras.losses.BinaryCrossentropy(),\n        optimizer=tf.keras.optimizers.SGD(learning_rate=0.001),\n        metrics=[tf.keras.metrics.AUC()])\n\nmultilingual_bert.summary()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"display_name":"Python 3","language":"python","name":"python3"},"language_info":{"codemirror_mode":{"name":"ipython","version":3},"file_extension":".py","mimetype":"text/x-python","name":"python","nbconvert_exporter":"python","pygments_lexer":"ipython3","version":"3.6.6"}},"nbformat":4,"nbformat_minor":4}