{"cells":[{"metadata":{},"cell_type":"markdown","source":"Hi! This notebook illustrates a simple way to make a multi-language of every transformer model by simply using XLM-R embedding, and then feed to the architecture you want (i.e. GPT2 in this notebook, so that you have XLM-GPT2), and then finetune it. \n\nThis notebook is about several months ago, and use a bit dated versions of TF and Transformers, so if you use the latest version, you may need to modify the code a bit. Please see Version 12 for the acutal running :)"},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q tensorflow==2.2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os\n\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom kaggle_datasets import KaggleDatasets\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\nimport gc\nfrom tensorflow.keras.mixed_precision import experimental as mixed_precision\n\nMIX = False\n\nif MIX:\n    tf.config.optimizer.set_jit(True)\n    policy = mixed_precision.Policy('mixed_bfloat16')\n    mixed_precision.set_policy(policy)\n#     tf.config.optimizer.set_experimental_options({\"auto_mixed_precision\": True})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tf.__version__)\nprint(transformers.__version__)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper Functions"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"def fix_fast(ids):\n    ids2 = [xx+1 for xx in ids]\n    return [0] + ids2 +[2]\n\ndef fast_encode(texts, tokenizer, chunk_size=256, maxlen=384):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([fix_fast(enc.ids) for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_xlmr(transformer, max_len=512):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    out = Dense(1, activation='sigmoid', dtype='float32')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    opt = Adam(lr=1e-5)\n    if MIX:\n        opt = tf.keras.mixed_precision.experimental.LossScaleOptimizer(opt, 'dynamic')\n    model.compile(opt, loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC(),'accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Configs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Data access\n# GCS_DS_PATH = KaggleDatasets().get_gcs_path()\n\n# Configuration\nEPOCHS = 3\nLR = 3e-4\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync\n\nif MIX:\n    BATCH_SIZE = 32 * strategy.num_replicas_in_sync\n\nprint(BATCH_SIZE)\n    \nMAX_LEN = 192\n# MODEL = '../input/mlm-epoch3-ppl505'\nMODEL = '../input/mlm-epoch2-ppl469' #'jplu/tf-xlm-roberta-large'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Create fast tokenizer"},{"metadata":{"trusted":true},"cell_type":"code","source":"# First load the real tokenizer \ntokenizer = AutoTokenizer.from_pretrained(MODEL,use_fast=True)\n\nfrom tokenizers import SentencePieceBPETokenizer\nfast_tokenizer = SentencePieceBPETokenizer('../input/mlm-epoch1-ppl583/xlmr_vocab.json', '../input/mlm-epoch1-ppl583/xlmr_merges.txt')\n\nfast_tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"text = \"Hello my name is Jung Прежде всего, это было хорошее Você é especialista? Você não pode\"\nprint(tokenizer.encode(text))\nprint(fast_tokenizer.encode(text).ids) # fast tokenizer cannot be used directly\nids2 = fix_fast(fast_tokenizer.encode(text).ids)\nprint(ids2)\nprint(len(text.split()),\n      len(tokenizer.encode(text)), \n      len(ids2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tokenizer.decode(tokenizer.encode(text)))\nprint(tokenizer.decode(ids2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load text data into memory"},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n\n# option1\ntrain2.toxic = train2.toxic.round().astype(int) \n# option2\n# train2.loc[train2.toxic >= 0.5,'toxic'] = 1\n# train2.loc[train2.toxic < .5,'toxic'] = 0\n# train2.toxic = train2.toxic.astype(int)\n\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid = valid.sample(frac=1, random_state=0)\n\ntrain = pd.concat([\n    train1[['comment_text', 'toxic']], #.sample(n=50000, random_state=0)\n    train2[['comment_text', 'toxic']].query('toxic==1'),\n    train2[['comment_text', 'toxic']].query('toxic==0').sample(n=100000, random_state=0)\n])\nprint(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n# x_train = np.load('/kaggle/input/jigsaw20-private-data/train_en_full2019.npz')['x'] # (1902194, 192) # from 2019 only\nx_train = np.load('/kaggle/input/jigsaw20-tpu-xlm-roberta/train_en.npz')['x'] # (1000000, 192) # from 2019 only\ny_train = np.load('/kaggle/input/jigsaw20-tpu-xlm-roberta/train_en.npz')['y'] #\nx_valid = np.load('/kaggle/input/jigsaw20-tpu-xlm-roberta/valid_en.npz')['x'] # (8000, 192)\ny_valid = np.load('/kaggle/input/jigsaw20-tpu-xlm-roberta/valid_en.npz')['y'] #\nx_test = np.load('/kaggle/input/jigsaw20-tpu-xlm-roberta/test_en.npz')['x'] # (63812, 192)\n\nprint(x_train.shape, x_valid.shape, x_test.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(y_train[:5])\ny_train[y_train < 0.5] = 0\ny_train[y_train >= 0.5] = 1\ny_train = y_train.astype(np.int32)\nprint(y_train[:5], len(y_train), len(y_train[y_train == 1]))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_xtest_new(x_test, batch=BATCH_SIZE):\n    '''\n    Ensure that x_test_new can be divided by BATCH_SIZE\n    '''\n    orig_len = len(x_test)\n    new_len = (orig_len//batch + 1)*(batch)\n    new_shape = list(x_test.shape)\n    new_shape[0] = new_len\n        \n    x_test_new = np.ones(new_shape)\n    x_test_new[:orig_len] = x_test\n    return x_test_new.astype(np.int32), orig_len","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test_new, orig_len = create_xtest_new(x_test, batch=BATCH_SIZE)\nprint(x_test_new.shape, orig_len)\n# print(x_test_new[-1], x_test_new[0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE, drop_remainder=True)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test_new)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build datasets objects"},{"metadata":{},"cell_type":"markdown","source":"## Load model into the TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"wpath = '../input/xlmr-test/xlmr_large_fixed_embed.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nfrom transformers import GPT2Tokenizer, TFGPT2LMHeadModel, TFGPT2Model\ntokenizerg = GPT2Tokenizer.from_pretrained('gpt2',pad_token=' ')\ntokenizerg.save_pretrained('.')\n\nwith strategy.scope():\n    xlmr_layer = TFAutoModel.from_pretrained(MODEL, from_pt=True) #TFAutoModel.from_pretrained(MODEL)\n    xlmr_layer.layers[0].embeddings.trainable = False # add one line\n    \n    gpt2_layer = TFGPT2Model.from_pretrained('gpt2') #TFGPT2LMHeadModel.from_pretrained('gpt2-medium')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(MODEL, wpath)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Now combine everything to make XLM-GPT2 TF Model!!!"},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import *\nfrom tensorflow.keras import Model\n\nclass XLMTransformers(tf.keras.Model):\n    def __init__(self, emb_layer, body_layer, connect_dim=None, dropout_rate=0.2, batch_size=BATCH_SIZE//strategy.num_replicas_in_sync):\n        super().__init__()\n        self.emb_layer = emb_layer\n        self.body_layer = body_layer\n        self.connect_dim = connect_dim\n        \n        if self.connect_dim is None:\n            conf = self.body_layer.layers[-1].get_config()\n            # TODO : config are different for each arch, below works only for GPT2 but not Albert\n            self.connect_dim = conf['transformers_config']['n_embd'] \n\n        if self.connect_dim > 0:\n            self.connect_layer = Dense(self.connect_dim,activation='linear') # TODO : identity initializer or ?\n\n        self.pooling_layer = GlobalMaxPooling1D()\n        self.drop_layer = Dropout(dropout_rate)\n        self.pred_layer = Dense(1,activation='sigmoid')\n        \n        self.batch_size = batch_size\n        \n    def call(self, input_ids):\n        pos_ids = self.emb_layer.create_position_ids_from_input_ids(input_ids)\n        token_type_ids = tf.zeros([self.batch_size, input_ids.shape[1]])\n        \n        x = self.emb_layer([input_ids, pos_ids, token_type_ids, None])\n        if self.connect_dim > 0:\n            x = self.connect_layer(x)\n        x = self.body_layer({'input_ids':None, 'inputs_embeds':x})[0]\n        x = self.pooling_layer(x)\n        x = self.drop_layer(x)\n        x = self.pred_layer(x)\n        return x\n    \n    def predict_numpy(self, x_test, batch=BATCH_SIZE):\n        '''\n        Purpose: \n          just to make sure that x_test_new can be divided by self.batch_size\n          Currently, unusable with unknown reason\n        ''' \n        x_test_new, orig_len = create_xtest_new(x_test)\n        pred = self.predict(x_test_new.astype(np.int32), verbose=1)\n        \n        return pred[:orig_len]\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"text = \"Hello my name is Jung Прежде всего, это было хорошее Você é especialista? Você não pode asdf asdfljsd fj\"\nenc = tokenizer.batch_encode_plus([text], return_token_type_ids=True, return_attention_mask=False, pad_to_max_length=True)\n\nwith strategy.scope():\n    xlm_gpt2 = XLMTransformers(emb_layer = xlmr_layer.layers[0].embeddings, \n                           body_layer = gpt2_layer)\n    \n    y = xlm_gpt2(tf.constant(enc['input_ids']))\n    xlm_gpt2.compile(Adam(lr=LR), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC()])\n\nprint(y.shape)\n# xlm_gpt2.summary()\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model"},{"metadata":{},"cell_type":"markdown","source":"First, we train on the subset of the training set, which is completely in English."},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_train.shape[0] // BATCH_SIZE\ntrain_history = xlm_gpt2.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xlm_gpt2.evaluate(valid_dataset) # ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xlm_gpt2.save_weights('xlm_gpt2_small_fixed_embed_finetuned_en_only.h5')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now that we have pretty much saturated the learning potential of the model on english only data, we train it for one more epoch on the `validation` set, which is significantly smaller but contains a mixture of different languages."},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = xlm_gpt2.fit(\n    valid_dataset.shuffle(2048).repeat(),\n    steps_per_epoch=n_steps,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"## note that test_dataset contains more elements than original\npred = xlm_gpt2.predict(test_dataset, \n                        verbose=1)\nprint(pred.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = pred[:orig_len]\nsub.to_csv('submission.csv', index=False)\nsub.tail()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}