{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"MAX_LEN = 192  #Reduced for quicker execution\nLR = 1e-5\nBATCH_SIZE = 16 # per TPU core\nTOTAL_STEPS_STAGE1 = 300\nVALIDATE_EVERY_STAGE1 = 100\nTOTAL_STEPS_STAGE2 = 200\nVALIDATE_EVERY_STAGE2 = 100\n\nPRETRAINED_MODEL = 'jplu/tf-xlm-roberta-large'\nD = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'\n\nimport os\nimport numpy as np\nimport pandas as pd\nfrom sklearn.metrics import roc_auc_score\nimport tensorflow as tf\nprint(tf.__version__)\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.models import Model\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nimport logging\n# no extensive logging \nlogging.getLogger().setLevel(logging.NOTSET)\n\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def connect_to_TPU():\n    \"\"\"Detect hardware, return appropriate distribution strategy\"\"\"\n    try:\n        # TPU detection. No parameters necessary if TPU_NAME environment variable is\n        # set: this is always the case on Kaggle.\n        tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n        print('Running on TPU ', tpu.master())\n    except ValueError:\n        tpu = None\n\n    if tpu:\n        tf.config.experimental_connect_to_cluster(tpu)\n        tf.tpu.experimental.initialize_tpu_system(tpu)\n        strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    else:\n        # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n        strategy = tf.distribute.get_strategy()\n\n    global_batch_size = BATCH_SIZE * strategy.num_replicas_in_sync\n\n    return tpu, strategy, global_batch_size\n\n\ntpu, strategy, global_batch_size = connect_to_TPU()\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_df = pd.read_csv(D+'jigsaw-toxic-comment-train.csv')\nval_df = pd.read_csv(D+'validation.csv')\ntest_df = pd.read_csv(D+'test.csv')\nsub_df = pd.read_csv(D+'sample_submission.csv')\n\n# subsample the train dataframe to 50%-50%\ntrain_df = pd.concat([\n    train_df.query('toxic==1'),\n    train_df.query('toxic==0').sample(sum(train_df.toxic),random_state=42)\n])\n# shufle it just to make sure\ntrain_df = train_df.sample(frac=1, random_state = 42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndef regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids'])\n    \n\ntokenizer = AutoTokenizer.from_pretrained(PRETRAINED_MODEL)\nX_train = regular_encode(train_df.comment_text.values, tokenizer, maxlen=MAX_LEN)\nX_val = regular_encode(val_df.comment_text.values, tokenizer, maxlen=MAX_LEN)\nX_test = regular_encode(test_df.content.values, tokenizer, maxlen=MAX_LEN)\n\ny_train = train_df.toxic.values.reshape(-1,1)\ny_val = val_df.toxic.values.reshape(-1,1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_dist_dataset(X, y=None, training=False):\n    dataset = tf.data.Dataset.from_tensor_slices(X)\n\n    ### Add y if present ###\n    if y is not None:\n        dataset_y = tf.data.Dataset.from_tensor_slices(y)\n        dataset = tf.data.Dataset.zip((dataset, dataset_y))\n        \n    ### Repeat if training ###\n    if training:\n        dataset = dataset.shuffle(len(X)).repeat()\n\n    dataset = dataset.batch(global_batch_size).prefetch(AUTO)\n\n    ### make it distributed  ###\n    dist_dataset = strategy.experimental_distribute_dataset(dataset)\n\n    return dist_dataset\n    \n    \ntrain_dist_dataset = create_dist_dataset(X_train, y_train, True)\nval_dist_dataset   = create_dist_dataset(X_val)\ntest_dist_dataset  = create_dist_dataset(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ndef create_model_and_optimizer():\n    with strategy.scope():\n        transformer_layer = TFAutoModel.from_pretrained(PRETRAINED_MODEL)                \n        model = build_model(transformer_layer)\n        optimizer = tf.keras.optimizers.Adam(learning_rate=LR, epsilon=1e-08)\n    return model, optimizer\n\n\ndef build_model(transformer):\n    inp = Input(shape=(MAX_LEN,), dtype=tf.int32, name=\"input_word_ids\")\n    # Huggingface transformers have multiple outputs, embeddings are the first one\n    # let's slice out the first position, the paper says its not worse than pooling\n    x = transformer(inp)[0][:, 0, :]  \n    out = Dense(1, activation='sigmoid')(x)\n    model = Model(inputs=[inp], outputs=[out])\n    \n    return model\n\n\nmodel, optimizer = create_model_and_optimizer()\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def define_losses_and_metrics():\n    with strategy.scope():\n        loss_object = tf.keras.losses.BinaryCrossentropy(\n            reduction=tf.keras.losses.Reduction.NONE, from_logits=False)\n\n        def compute_loss(labels, predictions):\n            per_example_loss = loss_object(labels, predictions)\n            loss = tf.nn.compute_average_loss(\n                per_example_loss, global_batch_size = global_batch_size)\n            return loss\n\n        train_accuracy_metric = tf.keras.metrics.AUC(name='training_AUC')\n\n    return compute_loss, train_accuracy_metric\n\n\n\ndef train(train_dist_dataset, val_dist_dataset=None, y_val=None,\n          total_steps=5000, validate_every=500):\n    step = 0\n    ### Training lopp ###\n    for tensor in train_dist_dataset:\n        distributed_train_step(tensor) \n        step+=1\n\n        if (step % validate_every == 0):   \n            ### Print train metrics ###  \n            train_metric = train_accuracy_metric.result().numpy()\n            print(\"Step %d, train AUC: %.5f\" % (step, train_metric))   \n            \n            ### Test loop with exact AUC ###\n            if val_dist_dataset:\n                val_metric = roc_auc_score(y_val, predict(val_dist_dataset))\n                print(\"     validation AUC: %.5f\" %  val_metric)   \n\n            ### Reset (train) metrics ###\n            train_accuracy_metric.reset_states()\n            \n        if step  == total_steps:\n            break\n\n\n\n@tf.function\ndef distributed_train_step(data):\n    strategy.experimental_run_v2(train_step, args=(data,))\n\ndef train_step(inputs):\n    features, labels = inputs\n\n    with tf.GradientTape() as tape:\n        predictions = model(features, training=True)\n        loss = compute_loss(labels, predictions)\n\n    gradients = tape.gradient(loss, model.trainable_variables)\n    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n\n    train_accuracy_metric.update_state(labels, predictions)\n\n\n\n\ndef predict(dataset):  \n    predictions = []\n    for tensor in dataset:\n        predictions.append(distributed_prediction_step(tensor))\n    ### stack replicas and batches\n    predictions = np.vstack(list(map(np.vstack,predictions)))\n    return predictions\n\n@tf.function\ndef distributed_prediction_step(data):\n    predictions = strategy.experimental_run_v2(prediction_step, args=(data,))\n    return strategy.experimental_local_results(predictions)\n\ndef prediction_step(inputs):\n    features = inputs  # note datasets used in prediction do not have labels\n    predictions = model(features, training=False)\n    return predictions\n\n\ncompute_loss, train_accuracy_metric = define_losses_and_metrics()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain(train_dist_dataset, val_dist_dataset, y_val,\n      TOTAL_STEPS_STAGE1, VALIDATE_EVERY_STAGE1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# make a new dataset for training with the validation data \n# with targets, shuffling and repeating\nval_dist_dataset_4_training = create_dist_dataset(X_val, y_val, training=True)\n\n# train again\ntrain(val_dist_dataset_4_training,\n      total_steps = TOTAL_STEPS_STAGE2, \n      validate_every = VALIDATE_EVERY_STAGE2)  # not validating but printing now","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nsub_df['toxic'] = predict(test_dist_dataset)[:,0]\nsub_df.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}