{"cells":[{"metadata":{"id":"joAjR5FSEEgD","outputId":"f8053f35-c4b7-4625-f0af-10418ef8d48f","trusted":true},"cell_type":"code","source":"%pip install transformers\n# %pip install tensor2tensor\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"osHmlEEVDkPZ"},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"cV91TW3yDkPd","outputId":"41452bd1-e76d-4460-f2dd-d5dff41f0eed"},"cell_type":"code","source":"import os\nimport numpy as np\nimport pandas as pd\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint,EarlyStopping,ReduceLROnPlateau\ninterativeEnvironment=False\nlocalEnvironment=False\nIN_COLAB=False\ntry:\n    from kaggle_datasets import KaggleDatasets\nexcept ImportError as e:\n  try:\n    import google.colab\n    IN_COLAB = True\n  except:\n    IN_COLAB = False\n    localEnvironment=True\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\n# from transformers import XLMRobertaModel, XLMRobertaTokenizer, XLMRobertaConfig\n\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\nimport os\nos.environ['XLA_USE_BF16'] = \"1\"\nlocalEnvironment\npd.set_option('display.max_colwidth', -1)\npd.set_option('display.max_rows', 300)\npd.set_option('display.width', None)\nimport warnings\nimport matplotlib.pyplot as plt\n%matplotlib inline\nimport random\n# from pandarallel import pandarallel\n\n# pandarallel.initialize(nb_workers=4, progress_bar=True)\nfrom collections import Counter\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"kVzapwIwDkPh","outputId":"3799f69f-8d0e-40d5-8ce4-de043e5204e9"},"cell_type":"code","source":"def is_interactive():\n   return 'runtime' in get_ipython().config.IPKernelApp.connection_file\n\nprint('Interactive?', is_interactive())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"VeDaqoHADkPk"},"cell_type":"code","source":"# from pandarallel import pandarallel\n\n# pandarallel.initialize(nb_workers=2, progress_bar=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"e8UJwi0ODkPm"},"cell_type":"code","source":"tf_env=\"tensorflow\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"zCnZ4hmFDkPp"},"cell_type":"code","source":"fit_verbocity=1\nmodel=tf_env\nnrows=None\ninterativeEnvironment=is_interactive()\ninterativeEnvironment=False\nif localEnvironment:\n#     nrows=200\n    warnings.warn(\"Nrows limited\")\n    pass\nelif interativeEnvironment:\n    pass\nelse:\n    fit_verbocity=2\npooling_mode_cls_token=\"CLS_TOKEN\"\npooling_mode_fc=\"FC\"\npooling_mode=pooling_mode_fc\nfc_dims=[]\ntrainable_transformer=False\nuse_dann=False\ndann_lambda=-1\nuse_parcor=False\nparcor_lambda=1\nuse_augmented_data=False\nuse_lowercase_data=False\nuse_finetuning=True\nuse_validation_during_pretraining=True\nuse_translated_data=True\nuse_pretraining=True\nload_model=None","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"yBYWsxNuDkPr"},"cell_type":"code","source":"SEED = 42\n\n# BACKBONE_PATH = '../input/multitpu-inference'\n# CHECKPOINT_PATH = '../input/multitpu-inference/checkpoint-xlm-roberta.bin'\n\n\ndef seed_everything(seed):\n    random.seed(seed)\n    os.environ['PYTHONHASHSEED'] = str(seed)\n    np.random.seed(seed)\n    torch.manual_seed(seed)\n    torch.cuda.manual_seed(seed)\n    torch.backends.cudnn.deterministic = True\n    torch.backends.cudnn.benchmark = True\n\n# seed_everything(SEED)    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"ouDj1TAmDkPu"},"cell_type":"code","source":"@tf.custom_gradient\ndef grad_reverse(x):\n    y = tf.identity(x)\n    def custom_grad(dy):\n        return tf.math.scalar_mul(-1,dy)\n    return y, custom_grad\nclass GradReverse(tf.keras.layers.Layer):\n    def __init__(self):\n        super().__init__()\n\n    def call(self, x):\n        return grad_reverse(x)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"IYFfAta3DkP6","outputId":"0b640547-d970-405c-958a-c028ded9e2be"},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"ykxWl6WRDkP8","outputId":"a1ef76e7-8c1c-4038-e506-ce4eefa5ad36"},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Data access\nif not localEnvironment  and not IN_COLAB:\n    GCS_DS_PATH = KaggleDatasets().get_gcs_path(\"jigsaw-multilingual-toxic-comment-classification\")\n\n# Configuration\nNEXAMPLESPEREPOCH=240000\nEPOCHS = 10\nif strategy.num_replicas_in_sync==1:\n    BATCH_SIZE = 4 * strategy.num_replicas_in_sync\nelse:\n    BATCH_SIZE = 16 * strategy.num_replicas_in_sync\n\nMAX_LEN = 192\n# MAX_LEN = 512\n\nMODEL = 'jplu/tf-xlm-roberta-large'\nif localEnvironment:\n    MODEL='jplu/tf-xlm-roberta-base'\nif IN_COLAB:\n  MODEL='jplu/tf-xlm-roberta-base'\nprint(\"BATCH_SIZE: \", BATCH_SIZE)","execution_count":null,"outputs":[]},{"metadata":{"id":"rMU9J_akEoE3","trusted":true},"cell_type":"code","source":"if IN_COLAB:\n  project_id=\"global-sun-279412\"","execution_count":null,"outputs":[]},{"metadata":{"id":"bkUGjss8EpQn","outputId":"60582023-985e-453e-8479-8ae72950019d","trusted":true},"cell_type":"code","source":"!gcloud config set project {project_id}","execution_count":null,"outputs":[]},{"metadata":{"id":"4Yl9I-tZEpKR","trusted":true},"cell_type":"code","source":"bucket_name=\"jigsaw-tfrecords\"","execution_count":null,"outputs":[]},{"metadata":{"id":"i8w-Nn6eEpCC","trusted":true},"cell_type":"code","source":"if IN_COLAB:\n  from google.colab import auth\n  auth.authenticate_user()","execution_count":null,"outputs":[]},{"metadata":{"id":"DLhqzOlfElIj","trusted":true},"cell_type":"code","source":"# if IN_COLAB:\n#   from google.colab import drive\n#   drive.mount('/content/gdrive')\n","execution_count":null,"outputs":[]},{"metadata":{"id":"bKjnL40mE04O","outputId":"e0fc0c64-5ae4-470c-e255-be2c201c4d27","trusted":true},"cell_type":"code","source":"!gsutil ls gs://{bucket_name}/","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"bwFHPJJxDkP-"},"cell_type":"code","source":"# # First load the real tokenizer\n# tokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"EHV36-xLDkQB","outputId":"4e38a2fe-14b5-4480-8d0c-6b795dfbb400"},"cell_type":"code","source":"if localEnvironment:\n    root =\"..\"\nelif IN_COLAB:\n    root='gs://jigsaw-tfrecords/'\nelse:\n    root=\"/kaggle\"\nroot","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"oHPtbZgcDkQQ","outputId":"f67932a9-f42e-405d-9723-ef678ec369ca"},"cell_type":"code","source":"# validation_fold_index=random.choice(range(5))\nvalidation_fold_index=1\nvalidation_fold_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"2TmLgfZnDkQS"},"cell_type":"code","source":"def validationFoldRemoveFilter(validation_fold_index):\n    def _filter(features):\n#         label=example[\"toxic\"]\n#         print(label)\n        return tf.math.not_equal(features[\"ShardIndex\"],validation_fold_index)\n    return _filter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"NKOzygk9DkQU"},"cell_type":"code","source":"def validationFoldOnlyFilter(validation_fold_index):\n    def _filter(features):\n#         label=example[\"toxic\"]\n#         print(label)\n        return tf.math.equal(features[\"ShardIndex\"],validation_fold_index)\n    return _filter","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"5CbhkpuSDkQX","outputId":"3997b9c4-89e3-46cc-f143-acb0b9d1f23b"},"cell_type":"code","source":"if IN_COLAB:\n    TF_DS_PATH=root+\"jigsaw-official\"\nelif not localEnvironment:\n    TF_DS_PATH = KaggleDatasets().get_gcs_path(\"jigsawaugmented\")+\"/tfrecords\"\n#     if use_augmented_data:\n#         TF_DS_PATH = KaggleDatasets().get_gcs_path(\"jigsawaugmented\")+\"/tfrecords\"\n#     else:\n#         TF_DS_PATH = KaggleDatasets().get_gcs_path(\"jigsaw-as-tf-record\")\nelse:\n    TF_DS_PATH = \"../tfrecords\"\nTF_DS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file=\"en1_train_non_toxic_PropercaseSentences_0_4.tfrecord\"\n(\"Sentences\" not in file) or (\"LowercaseSentences\" in file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"id":"KPw6AjA2DkQZ","outputId":"4530fd20-7371-4aaf-d90c-c5748bab351a"},"cell_type":"code","source":"\ndata_files=tf.io.gfile.glob(TF_DS_PATH + \"/*.tfrecord\")\nif use_augmented_data:\n    pass\nelif use_lowercase_data:\n    data_files=list(filter(lambda file:(\"Sentences\" not in file) or (\"LowercaseSentences\" in file),data_files))\nelse:\n    data_files=list(filter(lambda file:\"Sentences\" not in file,data_files))\ndata_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"t5bti4vMDkQb","outputId":"243f3e6b-a0d8-4967-c5e5-2ee7007103b4"},"cell_type":"code","source":"toxic_en_files=list(filter(lambda file:\"nen\" not in file and \"toxic\" in file and \"non_toxic\" not in file and \"valid\" not in file and \"sharded\" not in file, data_files))\ntoxic_en_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":true,"id":"OXKrGJf4DkQd","outputId":"ebfea8dc-ecf9-47dd-ab78-a4f4d0234b15"},"cell_type":"code","source":"non_toxic_en_files=list(filter(lambda file:\"nen\" not in file and \"non_toxic\" in file and \"valid\" not in file and \"sharded\" not in file, data_files))\nnon_toxic_en_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"tTgDTRkeDkQf","outputId":"ec75a885-0ed3-4c9d-a9de-1d9fcd7f0170"},"cell_type":"code","source":"non_toxic_nen_files=list(filter(lambda file:\"non_toxic\" in file and \"nen1\" in file, data_files))\nnon_toxic_nen_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"4tREf8GFDkQh","outputId":"d89df30c-2a8f-434a-f236-13336f1389f8"},"cell_type":"code","source":"toxic_nen_files=list(filter(lambda file:\"non_toxic\" not in file and \"nen1\" in file, data_files))\ntoxic_nen_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"FfYTzTgYDkQj","outputId":"9541f19e-3cbd-409d-92de-d361d8b56d54"},"cell_type":"code","source":"unknown_nen_files=list(filter(lambda file: \"test\" not in file and \"nen2\" in file, data_files))\nunknown_nen_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"YVxm474NDkQl","outputId":"04b67047-4aaa-43aa-bac3-23d665f9ef1c"},"cell_type":"code","source":"valid_files=list(filter(lambda file:\"test\" not in file and \"nen1\" in file and \"Sentences\" not in file, data_files)) \nvalid_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_translated_data:\n    if IN_COLAB:\n      TRANLSATED_DATA_PATH_ONE=root+\"/translated-train-bias-all-langs\"\n      TRANLSATED_DATA_PATH_TWO=root+\"/jigsaw-train-multilingual-coments-google-api\"\n    elif not localEnvironment:\n        TRANSLATED_DATA_PATH = KaggleDatasets().get_gcs_path(\"jigsawtranslated\")+\"/tfrecords-translated\"\n\n    else:\n        pass\n    TRANSLATED_DATA_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"translated_files=[]\nif use_translated_data:\n    translated_files=tf.io.gfile.glob(TRANSLATED_DATA_PATH + \"/*tfrecord\")\ntranslated_files[:3],TRANSLATED_DATA_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic_translated_files=list(filter(lambda file:\"nen\" not in file and \"toxic\" in file and \"non\" not in file and \"valid\" not in file and \"sharded\" not in file, translated_files))\nrandom.choice(toxic_translated_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"non_toxic_translated_files=list(filter(lambda file:\"nen\" not in file and \"toxic\" in file and \"non\" in file and \"valid\" not in file and \"sharded\" not in file, translated_files))\nrandom.choice(non_toxic_translated_files)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tf.data.experimental.CsvDataset(\n#     \"my_file*.csv\",\n#     [tf.float32,  # Required field, use dtype or empty tensor\n#      tf.constant([0.0], dtype=tf.float32),  # Optional field, default to 0.0\n#      tf.int32,  # Required field, use dtype or empty tensor\n#      ],\n#     select_cols=[1,2,3]  # Only parse last three columns\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"yhtOQkbMDkQn","outputId":"31585943-ef8b-4995-c107-366daa7077df"},"cell_type":"code","source":"opus_langs=[['en', 'it'],\n ['en', 'ru'],\n ['en', 'tr'],\n ['en', 'pt_br'],\n ['en', 'fr'],\n ['en', 'pt'],\n ['en', 'es'],\n ['en', 'hi']]\nif IN_COLAB:\n  OPUS_DS_PATH=root+\"opus\"\nelif not localEnvironment:\n    OPUS_DS_PATH = KaggleDatasets().get_gcs_path(\"opus-for-jigsaw-sharded\")\nelse:\n    OPUS_DS_PATH = \"../opus-processed\"\nOPUS_DS_PATH","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"mYqPebnrDkQp"},"cell_type":"code","source":"if IN_COLAB:\n  extension=\".gz\"\nelse:\n  extension=\"\"\nopus_files=tf.io.gfile.glob(OPUS_DS_PATH + \"/*.tfrecord\"+extension)\nif interativeEnvironment:\n  print(opus_files)\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"4xdy0lJODkQu"},"cell_type":"code","source":"# Create a description of the features.\nfeature_description = {\n    'tokens': tf.io.FixedLenFeature([192], tf.int64),\n    'mask': tf.io.FixedLenFeature([192], tf.int64),\n#     'types': tf.io.FixedLenFeature([192], tf.int64, default_value=[0]*192),\n    'toxic': tf.io.FixedLenFeature([], tf.float32),\n    's_toxic': tf.io.FixedLenFeature([], tf.float32),\n    'source':tf.io.FixedLenFeature([], tf.int64),\n    'toxicity_annotator_count':tf.io.FixedLenFeature([], tf.int64),\n    'lang':tf.io.FixedLenFeature([], tf.string),\n    'ShardIndex':tf.io.FixedLenFeature([], tf.int64),\n}\n\ndef _parse_proto(example_proto):\n#     tf.print(example_proto)\n  # Parse the input `tf.Example` proto using the dictionary above.\n    example= tf.io.parse_single_example(example_proto, feature_description)\n#     return example\n    return example\n\n\ndef _form_tuple(example):\n  y={\"toxic\":tf.cast(example['s_toxic'],tf.float32)}\n  if use_dann:\n    y[\"lang\"]=tf.cast(tf.math.equal(example['lang'],'en'),tf.float32)\n  if use_parcor:\n    y[\"is_same\"]=example[\"is_same\"]\n\n\n  return (example['tokens'],example['mask']),y\n\ndef smoother(smoothing=0.01):\n    def smooth(example):\n\n        label=tf.cast(example['s_toxic'],tf.float32)\n#         tf.print(label)\n        label=tf.cond(label>=1.0-smoothing,lambda :tf.ones_like(label)-smoothing,lambda :tf.cond(label<0.0+smoothing,lambda :tf.zeros_like(label)+smoothing,lambda :label))\n        example['s_toxic']=label\n        return example\n    return smooth\ndef isValidLabel(example):\n    example['isLabelValid']=tf.cond(example['toxic']==-1,lambda :tf.constant(0.0),lambda :tf.constant(1.0))\n    return example","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"StofIE5fDkQw"},"cell_type":"code","source":"def perFileDataset(filename):\n    tf.print(\"Retrieving from filename\",filename)\n    return tf.data.TFRecordDataset(filename).map(_parse_proto)\ndef perFileDatasetShuffled(filename):\n    return perFileDataset(filename).shuffle(200000)\n\ndef unsqueezeTargetDimensions(features,labels):\n    labels=tf.cast(labels,tf.float32)\n    return features,tf.stack([tf.math.subtract(tf.constant([1.0]),labels),labels],axis=-1),","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"e2PbrkY9DkQ3"},"cell_type":"code","source":"def toxic_train_en_dataset(shardIndex=-1):\n    return tf.data.Dataset.from_tensor_slices(toxic_en_files).shuffle(2048).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12).map(smoother())\ndef toxic_train_translated_dataset(shardIndex=-1):\n    return tf.data.Dataset.from_tensor_slices(toxic_translated_files).shuffle(2048*4).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12).map(smoother(0.1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"60avbTO9DkQ5"},"cell_type":"code","source":"def non_toxic_train_en_dataset(shardIndex=-1):\n   return tf.data.Dataset.from_tensor_slices(non_toxic_en_files).shuffle(2048).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12).map(smoother())\ndef non_toxic_train_translated_dataset(shardIndex=-1):\n    return tf.data.Dataset.from_tensor_slices(non_toxic_translated_files).shuffle(2048*4).interleave(perFileDataset, num_parallel_calls=AUTO,cycle_length=12).map(smoother(0.1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"uT97GcsDDkQ7"},"cell_type":"code","source":"def toxic_train_nen_dataset(shardIndex=-1):\n    return tf.data.Dataset.from_tensor_slices(toxic_nen_files).shuffle(2048).interleave(perFileDataset).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother()).shuffle(8000)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"FWJnpCdcDkQ9"},"cell_type":"code","source":"def non_toxic_train_nen_dataset():\n    return tf.data.Dataset.from_tensor_slices(non_toxic_nen_files).shuffle(2048).interleave(perFileDataset).filter(validationFoldRemoveFilter(validation_fold_index)).map(smoother()).shuffle(8000)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"5GW7sxEODkRA"},"cell_type":"code","source":"def unknown_nen_dataset():\n    return tf.data.Dataset.from_tensor_slices(unknown_nen_files).shuffle(2048).interleave(perFileDataset,num_parallel_calls=AUTO)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"hu9DGIkODkRC"},"cell_type":"code","source":"def valid_nen_dataset():\n    return tf.data.Dataset.from_tensor_slices(valid_files).interleave(perFileDataset).filter(validationFoldOnlyFilter(validation_fold_index))#.map(_form_tuple)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def finetune_nen_dataset():\n    return tf.data.Dataset.from_tensor_slices(valid_files).shuffle(2048).interleave(perFileDataset).filter(validationFoldRemoveFilter(validation_fold_index))#.map(_form_tuple)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"Rt0bu4FxDkRE","outputId":"45f716da-79a6-41ce-eae4-81cf975a5e16"},"cell_type":"code","source":"%%time\nif interativeEnvironment:\n    total_by_source=Counter()\n    total_by_toxic=Counter()\n    total=0\n    for i,l in tqdm(enumerate(non_toxic_train_nen_dataset())):\n        total+=1\n        total_by_source.update([l[\"source\"].numpy()])\n        total_by_toxic.update([l[\"s_toxic\"].numpy()])\n    total_nen_train_non_toxic=total\n    total,total_by_toxic.most_common(),total_by_source.most_common()\nelse:\n    total_nen_train_non_toxic=5956","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"SBVFkjD6DkRG","outputId":"ee51660a-39c9-4ed1-8c2f-b0d0f1334797"},"cell_type":"code","source":"%%time\nif interativeEnvironment:\n    total_by_source=Counter()\n    total_by_toxic=Counter()\n    total=0\n    for i,l in tqdm(enumerate(toxic_train_nen_dataset())):\n        total+=1\n        total_by_source.update([l[\"source\"].numpy()])\n        total_by_toxic.update([l[\"s_toxic\"].numpy()])\n    total_nen_train_toxic=total\n    total,total_by_toxic.most_common(),total_by_source.most_common()\nelse:\n    total_nen_train_toxic=1061","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"EiYQysOkDkRH","outputId":"6ffa07bb-43ab-4695-bc9f-cdf988bca33a"},"cell_type":"code","source":"%%time\nif interativeEnvironment:\n\n    total_by_source=Counter()\n    total_by_toxic=Counter()\n    total=0\n    for i,l in tqdm(enumerate(unknown_nen_dataset())):\n        total+=1\n        total_by_source.update([l[\"source\"].numpy()])\n        total_by_toxic.update([l[\"s_toxic\"].numpy()])\n    total_nen_unknown=total\n    total,total_by_toxic.most_common(),total_by_source.most_common()\nelse:\n    total_nen_unknown=71078","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"QRcvIon4DkRJ","outputId":"480c9e1c-4bb0-4048-9b54-4113b2fa2fdf"},"cell_type":"code","source":"n_steps = NEXAMPLESPEREPOCH // BATCH_SIZE\nnum_nen=0\nif use_validation_during_pretraining:\n    num_nen+=total_nen_train_toxic+total_nen_train_non_toxic\nif use_dann:\n    num_nen += total_nen_unknown\n# n_steps=20\n# valid_dataset=list(valid_dataset)[:20]\nn_steps,num_nen","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"A36z9eu_DkRN"},"cell_type":"code","source":"num_dataset_section=2\nif use_translated_data:\n    num_dataset_section+=2\ndef f0():\n    return unknown_nen_dataset().repeat().take(total_nen_unknown)\ndef f1():\n    return toxic_train_en_dataset().repeat().take(int((NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))\ndef f2():\n    return non_toxic_train_en_dataset().repeat().take(int((NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))\ndef f3():\n    return toxic_train_translated_dataset().repeat().take(int((NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))\ndef f4():\n    return non_toxic_train_translated_dataset().repeat().take(int((NEXAMPLESPEREPOCH-num_nen)/num_dataset_section))\ndef f5():\n    return toxic_train_nen_dataset()\ndef f6():\n    return non_toxic_train_nen_dataset()\nbranches={0:f0,1:f1,2:f2,3:f3,4:f4,5:f5,6:f6}\n    \n    \n\nmixedDatasetParams=[1,2]\nif use_dann:\n    mixedDatasetParams.append(0)\n    # mixedDatasetParams=[[\"en\",\"toxic\"]]#,[\"en\",\"nontoxic\"]]#,[\"nen\",\"nontoxic\"],[\"nen\",\"toxic\"]]\n#     mixedDatasetParams=[[\"nen\",\"unknown\"]]\nif use_validation_during_pretraining:\n    mixedDatasetParams.extend([5,6])\nif use_translated_data:\n    mixedDatasetParams.extend([3,4])\ndef mapParamsToDataset(params):\n    return tf.switch_case(params,branches)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"xaX-s5tVDkRO"},"cell_type":"code","source":"cycle_length=4\nif use_dann:\n  cycle_length+=1\ndef non_parallel_dataset():\n  return tf.data.Dataset.from_tensor_slices(mixedDatasetParams).interleave(mapParamsToDataset, num_parallel_calls=AUTO,cycle_length=cycle_length).take(NEXAMPLESPEREPOCH)\nif interativeEnvironment:\n  train_dict_dataset=non_parallel_dataset()\n  train_dict_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"non_parallel_dataset()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"taeZYQhKDkRR","outputId":"cda03030-eab2-4dc0-d4ae-62043fc45197"},"cell_type":"code","source":"%%time\nfrom collections import Counter\n# interativeEnvironment=True\nif interativeEnvironment:\n    dlist=train_dict_dataset.repeat()\n    epoch_metrics={}\n    shard_metrics={}\n    for i,l in tqdm(enumerate(dlist)):\n#         if i==200:\n#             break\n        epoch=str(i//NEXAMPLESPEREPOCH)\n        shard=str(i//40000)\n        if epoch==\"10\":\n            break\n        if epoch in epoch_metrics.keys():\n            pass\n        else:\n            total_by_source=Counter()\n            total_by_toxic=Counter()\n            total_by_toxic_source=Counter()\n            total_by_s_toxic=Counter()\n            total=0\n            epoch_metrics[epoch]={\"by_toxic\":total_by_toxic,\"by_source\":total_by_source,\"total\":total,\"by_both\":total_by_toxic_source,\"by_s_toxic\":total_by_s_toxic}\n        if shard in shard_metrics.keys():\n            pass\n        else:\n            stotal_by_source=Counter()\n            stotal_by_toxic=Counter()\n            stotal_by_toxic_source=Counter()\n            stotal_by_s_toxic=Counter()\n            stotal=0\n            shard_metrics[shard]={\"by_toxic\":stotal_by_toxic,\"by_source\":stotal_by_source,\"total\":stotal,\"by_both\":stotal_by_toxic_source,\"by_s_toxic\":stotal_by_s_toxic}    \n\n        total+=1\n        total_by_source.update([l[\"source\"].numpy()])\n        total_by_toxic.update([-1 if l[\"toxic\"] ==-1 else 1 if l[\"toxic\"].numpy()>=0.5 else 0])\n        total_by_s_toxic.update([-1 if l[\"s_toxic\"] ==-1 else 1 if l[\"s_toxic\"].numpy()>=0.5 else 0])\n        total_by_toxic_source.update([(round(l[\"toxic\"].numpy()),l[\"source\"].numpy())])\n        \n        \n        stotal+=1\n        stotal_by_source.update([l[\"source\"].numpy()])\n        stotal_by_toxic.update([-1 if l[\"toxic\"] ==-1 else 1 if l[\"toxic\"].numpy()>=0.5 else 0])\n        stotal_by_s_toxic.update([-1 if l[\"s_toxic\"] ==-1 else 1 if l[\"s_toxic\"].numpy()>=0.5 else 0])\n        stotal_by_toxic_source.update([(round(l[\"toxic\"].numpy()),l[\"source\"].numpy())])\n\n#         print(list(map(lambda a:a.numpy(),[l[\"source\"],l[\"ShardIndex\"],l[\"s_toxic\"]]))) \n    for epoch,epoch_metric in epoch_metrics.items():\n        print(\"Epoch:-\",epoch,\"Toxic:-\",epoch_metric[\"by_toxic\"].most_common(),\"By SToxic:\",epoch_metric[\"by_s_toxic\"].most_common(),\"By source:-\",epoch_metric[\"by_source\"].most_common(),\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"BUbBW_6JDkRU"},"cell_type":"code","source":"if interativeEnvironment:\n    for epoch,epoch_metric in shard_metrics.items():\n            print(\"Shard:-\",epoch,\"Toxic:-\",epoch_metric[\"by_toxic\"].most_common(),\"Stoxic:\",epoch_metric[\"by_s_toxic\"].most_common(),\"By source:-\",epoch_metric[\"by_source\"].most_common(),\"\\n\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"J7HnbZIEDkQs","outputId":"ed608df5-be03-4b77-f80d-fc199ba2bb2d"},"cell_type":"code","source":"def opus_files(lang):\n    return tf.io.gfile.glob(OPUS_DS_PATH + \"/{}_*.tfrecord\".format(lang)+extension)\nopus_files(\"es\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"am0mSjX9DkQy"},"cell_type":"code","source":"if use_parcor:\n  parallel_feature_description = {\n      'tokens_en': tf.io.FixedLenFeature([192], tf.int64),\n      'en_attn_mask': tf.io.FixedLenFeature([192], tf.int64),\n  #     'types': tf.io.FixedLenFeature([192], tf.int64, default_value=[0]*192),\n      'tokens_nen': tf.io.FixedLenFeature([192], tf.int64),\n      'nen_attn_mask': tf.io.FixedLenFeature([192], tf.int64),\n      'lang':tf.io.FixedLenFeature([], tf.string),\n      'source':tf.io.FixedLenFeature([], tf.int64,default_value=[5]),\n      's_toxic':tf.io.FixedLenFeature([2], tf.float32,default_value=[-1.0,-1.0]),\n      'is_same':tf.io.FixedLenFeature([], tf.float32,default_value=[1.0]),\n  }\n  def _parse_parallel_proto(example_proto):\n  #     tf.print(example_proto)\n    # Parse the input `tf.Example` proto using the dictionary above.\n      example= tf.io.parse_single_example(example_proto, parallel_feature_description)\n  #     return example\n      return example\n  if IN_COLAB:\n    compression_type=\"GZIP\"\n  else:\n    compression_type=\"\"\n  def perFileParallelDataset(filename,feature_description=feature_description):\n      tf.print(\"Retrieving from filename\",filename)\n      return tf.data.TFRecordDataset(filename,compression_type=compression_type).map(_parse_parallel_proto)\n\n  def make_parallel_pairs(example):\n      ret={}\n      # r=tf.random.uniform(shape=[])\n      # ret[\"tokens\"]=tf.cond(r>=0.5,lambda:tf.stack([example[\"tokens_en\"],example[\"tokens_nen\"]]),lambda :tf.stack([example[\"tokens_nen\"],example[\"tokens_en\"]])\n      ret[\"tokens\"]=tf.stack([example[\"tokens_en\"],example[\"tokens_nen\"]])\n      ret[\"mask\"]=tf.stack([example[\"en_attn_mask\"],example[\"nen_attn_mask\"]])\n      ret[\"s_toxic\"]=example[\"s_toxic\"]\n      ret[\"is_same\"]=example[\"is_same\"]\n      ret[\"lang\"]=example[\"lang\"]\n      return ret\n  def parallel_dataset(files):\n      \n      return tf.data.Dataset.from_tensor_slices(files).shuffle(2048).interleave(perFileParallelDataset).map(make_parallel_pairs)\n  def parallel_labelled_dataset(labelled_dataset):\n      return labelled_dataset.batch(2).map(make_parallel_labelled_pairs)\n  def make_parallel_labelled_pairs(example):\n      example[\"is_same\"]=tf.constant(0.0)\n      return example\n  langs=[\"es\",\"fr\",\"pt\",\"tr\",\"ru\",\"it\"]\n  langFiles=list(map(opus_files,langs))\n  def all_parallel_dataset():\n    return tf.data.Dataset.from_tensor_slices(langFiles).interleave(parallel_dataset, num_parallel_calls=AUTO,cycle_length=len(langs))\n  def parallel_nonparallel_mixer(params):\n      return tf.cond(params==\"parallel\",lambda :all_parallel_dataset().take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple),lambda :parallel_labelled_dataset(non_parallel_dataset()).take(int(NEXAMPLESPEREPOCH/4)).map(_form_tuple))\n  def parcor_dataset():\n      return tf.data.Dataset.from_tensor_slices([\"parallel\",\"nonparallel\"]).interleave(parallel_nonparallel_mixer,num_parallel_calls=2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"id":"IjX5bZXMDkQ1"},"cell_type":"code","source":"if interativeEnvironment:\n  ds=list(all_parallel_dataset().take(7))\n  print(list(map(lambda a:a[\"is_same\"],ds)))\n  # print(ds)\n  # print(ds[0].keys(),list(map(lambda a:a.shape,ds[0])))","execution_count":null,"outputs":[]},{"metadata":{"id":"4bDNI4NIXm7t","trusted":true},"cell_type":"code","source":"if interativeEnvironment:\n  ds=list(parallel_labelled_dataset(valid_nen_dataset()).take(7))\n  print(list(map(lambda a:a[\"is_same\"],ds)))\n","execution_count":null,"outputs":[]},{"metadata":{"id":"oFeE2ONukyXF","trusted":true},"cell_type":"code","source":"if interativeEnvironment:\n  list(parcor_dataset().take(2))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"H1MCttyYDkRW","outputId":"34ebe32b-355b-4b6c-fa68-67b18c45aef9"},"cell_type":"code","source":"\nif use_parcor:\n  train_dataset=parcor_dataset().shuffle(NEXAMPLESPEREPOCH).repeat().batch(int(BATCH_SIZE/2)).prefetch(AUTO)\n  valid_dataset=parallel_labelled_dataset(valid_nen_dataset()).map(_form_tuple).batch(int(BATCH_SIZE/2)).prefetch(AUTO)\n  finetune_dataset=parallel_labelled_dataset(finetune_nen_dataset()).map(_form_tuple).batch(int(BATCH_SIZE/2)).prefetch(AUTO)\nelse:\n  valid_dataset = valid_nen_dataset().map(_form_tuple).batch(BATCH_SIZE).prefetch(AUTO)\n  train_dataset=non_parallel_dataset().shuffle(NEXAMPLESPEREPOCH).map(_form_tuple).repeat().batch(BATCH_SIZE).prefetch(AUTO)\n  finetune_dataset=finetune_nen_dataset().shuffle(NEXAMPLESPEREPOCH).map(_form_tuple).batch(BATCH_SIZE).prefetch(AUTO)\nvalid_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"SWWhQ2MFDkRY"},"cell_type":"code","source":"# for ex,label,weight in valid_dataset:\n#     print(tf.shape(weight[0]))\n#     print(tf.shape(weight[1]))\n#     print(tf.shape(label[\"toxic\"]))\n#     print(tf.shape(label[\"lang\"]))\n#     break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"Ga1YL8EEDkPw"},"cell_type":"code","source":"from tensorflow.python.keras.utils import losses_utils\nclass WeightedBinaryCrossentropy(tf.keras.losses.BinaryCrossentropy):\n    def __init__(self,\n               from_logits=False,\n               label_smoothing=0,\n               reduction=losses_utils.ReductionV2.AUTO,lossWeight=0.1,\n               name='binary_crossentropy'):\n        \"\"\"Initializes `BinaryCrossentropy` instance.\n        Args:\n          from_logits: Whether to interpret `y_pred` as a tensor of\n            [logit](https://en.wikipedia.org/wiki/Logit) values. By default, we\n              assume that `y_pred` contains probabilities (i.e., values in [0, 1]).\n              **Note - Using from_logits=True may be more numerically stable.\n          label_smoothing: Float in [0, 1]. When 0, no smoothing occurs. When > 0,\n            we compute the loss between the predicted labels and a smoothed version\n            of the true labels, where the smoothing squeezes the labels towards 0.5.\n            Larger values of `label_smoothing` correspond to heavier smoothing.\n          reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to\n            loss. Default value is `AUTO`. `AUTO` indicates that the reduction\n            option will be determined by the usage context. For almost all cases\n            this defaults to `SUM_OVER_BATCH_SIZE`. When used with\n            `tf.distribute.Strategy`, outside of built-in training loops such as\n            `tf.keras` `compile` and `fit`, using `AUTO` or `SUM_OVER_BATCH_SIZE`\n            will raise an error. Please see this custom training [tutorial]\n            (https://www.tensorflow.org/tutorials/distribute/custom_training)\n            for more details.\n          name: (Optional) Name for the op. Defaults to 'binary_crossentropy'.\n        \"\"\"\n        super(WeightedBinaryCrossentropy, self).__init__(\n            name=name,\n            reduction=reduction,\n            from_logits=from_logits,\n            label_smoothing=label_smoothing)\n        self.lossWeight=lossWeight\n    def call(self, y_true, y_pred):\n        l= tf.math.scalar_mul(self.lossWeight,super().call(y_true,y_pred))\n#         tf.print(tf.shape(l))\n        return l","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"t3WHAzd0DkPz"},"cell_type":"code","source":"from tensorflow.python.keras.utils import losses_utils\nclass MaskedBinaryCrossentropy(tf.keras.losses.BinaryCrossentropy):\n    def __init__(self,\n               from_logits=False,\n               label_smoothing=0,\n               reduction=losses_utils.ReductionV2.AUTO,\n               name='binary_crossentropy'):\n        \"\"\"Initializes `BinaryCrossentropy` instance.\n        Args:\n          from_logits: Whether to interpret `y_pred` as a tensor of\n            [logit](https://en.wikipedia.org/wiki/Logit) values. By default, we\n              assume that `y_pred` contains probabilities (i.e., values in [0, 1]).\n              **Note - Using from_logits=True may be more numerically stable.\n          label_smoothing: Float in [0, 1]. When 0, no smoothing occurs. When > 0,\n            we compute the loss between the predicted labels and a smoothed version\n            of the true labels, where the smoothing squeezes the labels towards 0.5.\n            Larger values of `label_smoothing` correspond to heavier smoothing.\n          reduction: (Optional) Type of `tf.keras.losses.Reduction` to apply to\n            loss. Default value is `AUTO`. `AUTO` indicates that the reduction\n            option will be determined by the usage context. For almost all cases\n            this defaults to `SUM_OVER_BATCH_SIZE`. When used with\n            `tf.distribute.Strategy`, outside of built-in training loops such as\n            `tf.keras` `compile` and `fit`, using `AUTO` or `SUM_OVER_BATCH_SIZE`\n            will raise an error. Please see this custom training [tutorial]\n            (https://www.tensorflow.org/tutorials/distribute/custom_training)\n            for more details.\n          name: (Optional) Name for the op. Defaults to 'binary_crossentropy'.\n        \"\"\"\n        super(MaskedBinaryCrossentropy, self).__init__(\n            name=name,\n            reduction=reduction,\n            from_logits=from_logits,\n            label_smoothing=label_smoothing)\n    def call(self, y_true, y_pred):\n        # tf.print(\"toxic\",y_true)\n\n        if use_parcor:\n          y_true=tf.reshape(y_true,[-1,1])\n          y_pred=tf.reshape(y_pred,[-1,1])\n        # tf.print(\"ytrue\",tf.shape(y_true))\n        # tf.print(\"ypred\",tf.shape(y_pred))\n        y_pred=tf.squeeze(y_pred,axis =-1)\n        y_true=tf.squeeze(y_true,axis =-1)\n        mask=tf.math.greater_equal(y_true,0)\n        mask=tf.cast(mask,y_true.dtype)\n        \n        # tf.print(\"mask shape\",tf.shape(mask))\n        masked_y_true=mask * y_true\n        # tf.print(\"masked_y_true\",tf.shape(masked_y_true))\n        l=super().call(masked_y_true,y_pred)\n        # tf.print(\"pre mask loss\",tf.shape(l))\n        l=mask*l\n        # tf.print(\"post mask loss\",tf.shape(l))\n        # tf.print(\"toxic loss\",l)\n#         tf.print(tf.shape(l))\n        if use_parcor:\n          l=tf.reshape(l,[-1,2])\n          l=l[:,0]+l[:,1]\n        return l","execution_count":null,"outputs":[]},{"metadata":{"id":"llQv1RejNmWS","trusted":true},"cell_type":"code","source":"class SquareLoss(tf.keras.losses.Loss):\n  def __init__(self,reduction=losses_utils.ReductionV2.AUTO, name=\"squareLoss\",lossWeight=0):\n    super().__init__(reduction=reduction,name=name)\n    self.lossWeight=lossWeight\n  def call(self, y_true, y_pred):\n    # tf.print(\"is same pred\",y_pred)\n    # tf.print(\"is same true\",y_true)\n    is_same_masked= y_pred*y_true\n    loss=tf.math.scalar_mul(self.lossWeight,is_same_masked*is_same_masked)\n    # tf.print(\"is same loss\",loss)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"a5BQ7o1DDkP3"},"cell_type":"code","source":"\ndef build_model(transformer, max_len=512):\n    if use_parcor:\n      pair_input_word_ids = Input(shape=(2,max_len,), dtype=tf.int32, name=\"input_word_ids\")\n      pair_attention_masks = Input(shape=(2,max_len,), dtype=tf.int32, name=\"attention_mask\")\n      input_word_ids = tf.keras.backend.reshape(pair_input_word_ids,shape=(-1,max_len))\n      attention_masks= tf.keras.backend.reshape(pair_attention_masks,shape=(-1,max_len))\n      inputs=[pair_input_word_ids,pair_attention_masks]\n    else:\n      input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n      attention_masks = Input(shape=(max_len,), dtype=tf.int32, name=\"attention_mask\")\n      inputs=[input_word_ids,attention_masks]\n#     token_type_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"token_type_ids\")\n#     transformer.trainable = trainable_transformer\n#     cls_token=transformer((input_word_ids,attention_masks))[1]\n#     out = Dense(2, activation='sigmoid')(cls_token)\n    transformer_output = transformer((input_word_ids,attention_masks))\n    sequence_output = transformer_output[0]\n    transformer_output\n    if pooling_mode == pooling_mode_cls_token:\n        transformed_features = sequence_output[:, 0, :]\n        cls_token=transformed_features\n    else:\n        float_mask=tf.cast(tf.expand_dims(attention_masks,axis=-1),tf.float32)\n        lengths=tf.reduce_sum(float_mask,axis=-2)\n\n        masked_vals=tf.multiply(sequence_output,float_mask)\n        sums=tf.reduce_sum(masked_vals,axis=-2)\n        avgs=tf.math.divide(sums,lengths)\n        maxes=tf.math.reduce_max(masked_vals,axis=-2)\n        transformed_features=tf.concat([avgs,maxes],axis=-1)\n        cls_token=transformed_features\n    for dims in fc_dims:\n        cls_token=Dense(dims, activation='relu')(cls_token)\n    if use_dann:\n        lang_token=GradReverse()(transformed_features)\n        for dims in fc_dims:\n            lang_token=Dense(dims, activation='relu')(lang_token)\n        lang=Dense(1, activation=\"sigmoid\",name=\"lang\")(lang_token)\n        \n    \n    \n    if use_parcor:\n      toxic = Dense(1, activation=\"sigmoid\",name=\"unpair_toxic\")(cls_token)\n      toxic=tf.keras.backend.reshape(toxic,shape=(-1,2,1))\n      toxic=tf.keras.layers.Layer(name=\"toxic\")(toxic)\n      is_same=tf.keras.layers.Subtract(name=\"is_same\")([toxic[:,1,:],toxic[:,0,:]])\n    else:\n      toxic = Dense(1, activation=\"sigmoid\",name=\"toxic\")(cls_token)\n    out=[toxic]\n    if use_parcor:\n      out.append(is_same)\n    celoss=MaskedBinaryCrossentropy(\n        from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,\n        name='crossentropy_loss'\n    )\n    loss={\"toxic\":celoss}\n    if use_dann:\n        domainGap=WeightedBinaryCrossentropy(\n        from_logits=False, label_smoothing=0, reduction=losses_utils.ReductionV2.AUTO,\n        name='domaingap_loss',lossWeight=dann_lambda\n        )\n        out.append(lang)\n        loss[\"lang\"]=domainGap\n    if use_parcor:\n      loss[\"is_same\"]=SquareLoss(lossWeight=parcor_lambda)\n    model = Model(inputs=inputs, outputs=out)\n    model.compile(Adam(lr=5e-6), loss=loss, metrics={\"toxic\":['accuracy',tf.keras.metrics.AUC(name=\"auc\"),tf.keras.metrics.AUC(name=\"pr-auc\",curve=\"PR\")]})\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"mLDbVvt0DkRa"},"cell_type":"code","source":"def setTrainableLayers(transformer,embeddingsTrainable=False,poolerTrainable=False,hiddenTrainable=8):\n    transformer.layers[0].pooler.trainable=poolerTrainable\n    transformer.layers[0].embeddings.trainable=embeddingsTrainable\n    trainableHiddens=transformer.layers[0].encoder.layer[-1*hiddenTrainable:]\n    for layer in transformer.layers[0].encoder.layer:\n        if layer in trainableHiddens:\n            layer.trainable=True\n        else:\n            layer.trainable=False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"DKzB28y4DkRc","outputId":"98e38f51-a6de-4fb8-dec4-c8e0ce0d42dd"},"cell_type":"code","source":"%%time\n# tf.keras.backend.clear_session()\nwith strategy.scope():\n#     tf.keras.backend.clear_session()\n\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n#     setTrainableLayers(transformer_layer)\n\n    model = build_model(transformer_layer, max_len=MAX_LEN)\n    if load_model is not None:\n            model.load_weights(load_model)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"kq3F-nzJDkRd","outputId":"e95c9783-b3b7-4aac-ce5b-22655bffdb03"},"cell_type":"code","source":"MODEL","execution_count":null,"outputs":[]},{"metadata":{"id":"4tQYTDSgFS86","outputId":"f94887c4-dd54-4a51-9a81-ed2b7d9fe54a","trusted":true},"cell_type":"code","source":"from datetime import datetime\n\n# datetime object containing current date and time\nnow = datetime.now()\n \ndt_string = now.strftime(\"%d_%m_%Y_%H_%M_%S_\")\nout_path=dt_string+'zero_shot_pro.h5'\nwrite_root=\"\"\nif IN_COLAB:\n  write_root=root+\"outputs/\"\n  out_path=write_root+out_path\nprint(out_path)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"nyjxDgSrDkRf"},"cell_type":"code","source":"#checkpointing best\nif use_pretraining:\n    metric_prefix=\"\"\n    if use_parcor:\n        metric_prefix=\"_toxic\"\n    mc = ModelCheckpoint(out_path, monitor=\"val\"+metric_prefix+\"_auc\", mode='max',verbose=1,save_weights_only=True,save_best_only=True)\n    es = EarlyStopping(monitor=\"val\"+metric_prefix+\"_auc\", mode='max',patience=2,verbose=1,restore_best_weights=True)\n    lrScheduler=ReduceLROnPlateau(\n        monitor=\"val\"+metric_prefix+\"_loss\",\n        factor=0.2,\n        patience=1,\n        verbose=1,\n        mode=\"min\",\n        min_delta=0,\n        cooldown=0,\n        min_lr=1e-10,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"sEaXi6NaDkRh"},"cell_type":"code","source":"# if interativeEnvironment:\n# n_steps=4\n# EPOCHS=2\nif use_pretraining:\n    train_history = model.fit(\n        train_dataset,\n        steps_per_epoch=n_steps,\n        validation_data=valid_dataset,\n        epochs=EPOCHS,\n        verbose=1,\n        callbacks=[mc,es,lrScheduler]\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"GPk876S1DkRj"},"cell_type":"code","source":"if use_pretraining:\n\n    if use_parcor or use_dann:\n        metric_prefix=\"toxic_\"\n    else:\n        metric_prefix=\"\"\n    plt.plot(train_history.history[metric_prefix+'auc'])\n    plt.plot(train_history.history['val_'+metric_prefix+'auc'])\n\n    plt.title('Model auc')\n    plt.ylabel('Auc')\n    plt.xlabel('Epoch')\n    plt.legend(['Train', 'Test'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"BnSp6SeODkRk"},"cell_type":"code","source":"if use_pretraining:\n\n    if use_parcor or use_dann:\n        plt.plot(train_history.history['toxic_loss'])\n        plt.plot(train_history.history['val_toxic_loss'])\n    plt.plot(train_history.history['loss'])\n    plt.plot(train_history.history['val_loss'])\n    plt.title('Model Loss')\n    plt.ylabel('Loss')\n    plt.xlabel('Epoch')\n    plt.legend(['Toxic Train', 'Toxic Test','Train', 'Test'], loc='upper left')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"umBvHj5pDkRm"},"cell_type":"code","source":"%%time\n\n\nsub = pd.read_csv(root+'/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv',nrows=nrows)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"mMiu30fmDkRn"},"cell_type":"code","source":"test_files=list(filter(lambda file:\"test\"  in file and \"nen\" in file, data_files))\ntest_files","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"eD15hppVDkRo"},"cell_type":"code","source":"if use_parcor:\n    test_dataset=parallel_labelled_dataset(tf.data.Dataset.from_tensor_slices(test_files).interleave(perFileDataset,cycle_length=1)).map(_form_tuple).batch(int(BATCH_SIZE/2)).prefetch(AUTO)\nelse:\n    test_dataset=tf.data.Dataset.from_tensor_slices(test_files).interleave(perFileDataset,cycle_length=1).batch(BATCH_SIZE).map(_form_tuple).prefetch(AUTO)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"rX_ybJ5-DkRq"},"cell_type":"code","source":"if use_pretraining:\n    l=model.predict(test_dataset, verbose=1)\n    if use_dann:\n        l=l[0]\n    if use_parcor:\n        l=np.reshape(l,(-1,1))\n    sub['toxic'] = l[:,-1]\n    # sub['toxic'] = (l[:,1]+(1-l[:,0]))/2\n    if use_finetuning:\n        write_file=write_root+'submission.unfinetuned.csv'\n    else:\n        write_file=write_root+'submission.csv'\n    sub.to_csv(write_file, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_finetuning:\n    # datetime object containing current date and time\n    now = datetime.now()\n\n    dt_string = now.strftime(\"%d_%m_%Y_%H_%M_%S_\")\n    out_path=dt_string+'zero_shot_pro.h5'\n    write_root=\"\"\n    if IN_COLAB:\n      write_root=root+\"outputs/\"\n      out_path=write_root+out_path\n    print(out_path)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_finetuning:\n    #checkpointing best\n    metric_prefix=\"\"\n    if use_parcor:\n        metric_prefix=\"_toxic\"\n    mc = ModelCheckpoint(out_path, monitor=\"val\"+metric_prefix+\"_auc\", mode='max',verbose=1,save_weights_only=True,save_best_only=True)\n    es = EarlyStopping(monitor=\"val\"+metric_prefix+\"_auc\", mode='max',patience=2,verbose=1,restore_best_weights=True)\n    lrScheduler=ReduceLROnPlateau(\n        monitor=\"val\"+metric_prefix+\"_loss\",\n        factor=0.2,\n        patience=1,\n        verbose=1,\n        mode=\"min\",\n        min_delta=0,\n        cooldown=0,\n        min_lr=1e-10,\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_finetuning:\n    # if interativeEnvironment:\n    # n_steps=4\n    # EPOCHS=2\n    train_history = model.fit(\n        finetune_dataset,\n#         steps_per_epoch=n_steps,\n        validation_data=valid_dataset,\n        epochs=EPOCHS,\n        verbose=1,\n        callbacks=[mc,es,lrScheduler]\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if use_finetuning:\n    l=model.predict(test_dataset, verbose=1)\n    if use_dann:\n        l=l[0]\n    if use_parcor:\n        l=np.reshape(l,(-1,1))\n    sub['toxic'] = l[:,-1]\n    # sub['toxic'] = (l[:,1]+(1-l[:,0]))/2\n    write_file=write_root+'submission.csv'\n    sub.to_csv(write_file, index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"DUCkPILFDkRr"},"cell_type":"code","source":"from keras.backend.tensorflow_backend import set_session\nfrom keras.backend.tensorflow_backend import clear_session\nfrom keras.backend.tensorflow_backend import get_session\nimport gc\n\n# Reset Keras Session\ndef reset_keras():\n#     del model\n    tf.keras.backend.clear_session()\n    print(gc.collect()) # if it's done something you should see a number being outputted","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"JdMbsMSrDkRs"},"cell_type":"code","source":"# reset_keras()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"scrolled":false,"id":"w--MB8aCDkRu"},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}