{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Data processing\n\nThis Notebooks performs 2 tasks:\nCreate TF Records, which is a TPU-friendly format to take advantage of TPU's\nExport the generated files to GCS, to stream them to our models","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, time\nimport pandas as pd\nimport tensorflow as tf\nimport transformers as ppb\nimport tokenizers\nfrom tqdm import tqdm\nimport numpy as np\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\nimport seaborn as sns\nimport glob\nsns.set()\nimport logging\nlogging.getLogger().setLevel(logging.NOTSET)\nprint(tf.version.VERSION)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Set global variables\n\nSet maximum sequence length, model and path variables.","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"SEQUENCE_LENGTH = 128\nMODEL_NAME = 'jplu/tf-xlm-roberta-large' # The model you want to use here\nDATA_PATH =  \"../input/jigsaw-multilingual-toxic-comment-classification\"\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Datasets\nRead and extract relevant columns from datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train1_df = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain2_df = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\ntrain2_df.toxic = train2_df.toxic.round().astype(int)\nprint(\"Training Dataset 1 with {} samples\".format(len(train1_df)))\nprint(\"Training Dataset 2 with {} samples\".format(len(train2_df)))\ntrain_df = pd.concat([train1_df[['id','comment_text','toxic']], train2_df[['comment_text','toxic']]])\nprint(\"Training Dataset with {} samples\".format(len(train_df)))\ndel train1_df\ndel train2_df\n\nvalidation_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\nprint(\"Validation Dataset with {} samples\".format(len(validation_df)))\ntest_df = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nprint(\"Test Dataset with {} samples\".format(len(test_df)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenizer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tokenizer(model_name=MODEL_NAME):\n    tokenizer = ppb.AutoTokenizer.from_pretrained(model_name)\n    print(type(tokenizer))\n  \n    return tokenizer\n\ntokenizer = get_tokenizer()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing\n\nProcess individual sentences for input to BERT using the tokenizer, and then prepare the entire dataset. The same code will process the other training data files, as well as the validation and test data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\ndef process_sentence(sentence, max_seq_length, tokenizer):\n    \"\"\"Helper function to prepare data for any BERT model. Converts sentence input examples\n    into the form ['input_ids', 'input_mask', 'segment_ids'].\"\"\"\n    # Tokenize, and truncate to max_seq_length if necessary.\n    input_ids = tokenizer.encode(sentence)\n    if len(input_ids) > max_seq_length :\n        input_ids = input_ids[:max_seq_length]\n\n    # The mask has 1 for real tokens and 0 for padding tokens. Only real\n    # tokens are attended to.\n    input_mask = [1] * len(input_ids)\n\n    # Zero-pad up to the sequence length.\n    pad_length = max_seq_length - len(input_ids)\n    input_ids.extend([0] * pad_length)\n    input_mask.extend([0] * pad_length)\n\n    # We only have one input segment.\n    segment_ids = [0] * max_seq_length\n\n    return (input_ids, input_mask, segment_ids)\n\ndef _int64_feature(value):\n    \"\"\"Returns a single element int64_list from a bool / enum / int / uint.\"\"\"\n    return tf.train.Feature(int64_list=tf.train.Int64List(value=[value]))\n\ndef _int64_feature_list(value):\n    \"\"\"Returns an int64_list from a bool / enum / int / uint.\"\"\"\n    v = tf.train.Int64List(value=value)\n    return tf.train.Feature(int64_list=v)\n\ndef serialize_example(f0, f1, f2, y):\n    \"\"\"\n    Creates a tf.Example message ready to be written to a file.\n    \"\"\"\n    # Create a dictionary mapping the feature name to the tf.Example-compatible\n    # data type.    \n    feature = {\n      'f0': _int64_feature_list(f0), # input_word_ids\n      'f1': _int64_feature_list(f1), # masks\n      'f2': _int64_feature_list(f2), # input_type\n      'y':  _int64_feature(y), # target\n    }\n\n    # Create a Features message using tf.train.Example.\n    example_proto = tf.train.Example(features=tf.train.Features(feature=feature))\n    return example_proto.SerializeToString()\n\ndef preprocess_and_save_tfrec(dataframe, filename, text_label='comment_text', seq_length=SEQUENCE_LENGTH, records_per_file=200000, verbose=True):\n    \"\"\"Preprocess a CSV to the expected TF Dataset in TFRecord format, and save the result.\n    Google recommends each file to be ~100MB, so 200K samples per file is about 100MB\"\"\"\n    processed_filename = (filename.rstrip('.csv') + \"-processed-seqlen{}.csv\".format(SEQUENCE_LENGTH))\n\n    start = time.time()\n    \n    for i in range(0,len(dataframe), records_per_file):\n        processed_df = dataframe.iloc[i:i + records_per_file]\n\n        input_word_ids, input_mask, all_segment_id = (zip(*processed_df[text_label].apply(lambda x: process_sentence(x, SEQUENCE_LENGTH, tokenizer))))\n        y = processed_df['toxic'].values\n\n        with tf.io.TFRecordWriter(filename+'_'+str(i)+'.tfrec') as writer:\n          for k in range(len(processed_df)):\n            example = serialize_example(input_word_ids[k], input_mask[k], all_segment_id[k], y[k])\n            writer.write(example)\n\n        if verbose:\n            print('Processed {} examples in {}'.format(i + len(processed_df), time.time() - start))\n    return\n  \n# Process the datasets.\nprint('Transforming data and generating files, this could take a while...')\npreprocess_and_save_tfrec(train_df.iloc[:30000], 'train') # remove the .iloc to process all data\npreprocess_and_save_tfrec(validation_df, 'validation')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Example of how to read a TFRecord\nWe need to instantiate a TFRecordDataset object from a ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"filenames = glob.glob('*.tfrec')\nraw_dataset = tf.data.TFRecordDataset(filenames)\nraw_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for raw_record in raw_dataset.take(1):\n  example = tf.train.Example()\n  example.ParseFromString(raw_record.numpy())\n  print(example)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Export to GCS","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"This part is for creating and uploading automatically your generated files from Kaggle to GCS. You can obviously do it manually by downloading all your files and uploading to GCS using the web console. If you do not have a nice bandwidth you might want to use the following scrip.  \n\nFirst thing you need to generate is a credentials.json file from your GCS IAM section.\n\n**NOTE: DON'T MAKE THIS INFORMATION PUBLIC!!, otherwise anyone with this data could access your GC account**","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"This part uses GCS API: \n[https://googleapis.dev/python/storage/latest/index.html](http://)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import json\nfrom google.oauth2 import service_account\nfrom google.cloud import storage\nimport glob\n\n# Copy here your credentials.json data\nauth = {\n# your auth data comes here. This is sensitive data\n}\n\nwith open('auth.json', 'w') as json_file:\n  json.dump(auth, json_file)\n\ncredentials = service_account.Credentials.from_service_account_file('auth.json')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Utility functions to manage GCS\n\ndef create_bucket(dataset_name):\n    \"\"\"Creates a new bucket. https://cloud.google.com/storage/docs/ \"\"\"\n    bucket = storage_client.create_bucket(dataset_name)\n    print('Bucket {} created'.format(bucket.name))\n\ndef upload_blob(bucket_name, source_file_name, destination_blob_name):\n    \"\"\"Uploads a file to the bucket. https://cloud.google.com/storage/docs/ \"\"\"\n    bucket = storage_client.get_bucket(bucket_name)\n    blob = bucket.blob(destination_blob_name)\n    blob.upload_from_filename(source_file_name)\n    print('File {} uploaded to {}.'.format(\n        source_file_name,\n        destination_blob_name))\n    \ndef list_blobs(bucket_name):\n    \"\"\"Lists all the blobs in the bucket. https://cloud.google.com/storage/docs/\"\"\"\n    blobs = storage_client.list_blobs(bucket_name)\n    for blob in blobs:\n        print(blob.name)\n        \ndef download_to_kaggle(bucket_name,destination_directory,file_name):\n    \"\"\"Takes the data from your GCS Bucket and puts it into the working directory of your Kaggle notebook\"\"\"\n    os.makedirs(destination_directory, exist_ok = True)\n    full_file_path = os.path.join(destination_directory, file_name)\n    blobs = storage_client.list_blobs(bucket_name)\n    for blob in blobs:\n        blob.download_to_filename(full_file_path)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Time to generate a bucket in your Google Cloud Storage project","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_PROJECT = 'YOUR-GOOGLE-CLOUD-PROJECT'\n\nstorage_client = storage.Client(project=GCS_PROJECT, credentials=credentials)\nbucket_name = 'your-bucket-name' # this must be unique accross all bucket names, so it could be something like your_name_something_else       \ntry:\n    create_bucket(bucket_name)\nexcept:\n    print('Probably your bucket already existed or the name you selected is already taken')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Copy all generated .tfrec files (from Kaggle output directory) to your GCS bucket","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"files = glob.glob('*.tfrec')\nfor f in files:\n    upload_blob(bucket_name, f, f)\n    \nprint('Listing data in {} :'.format(bucket_name))   \nlist_blobs(bucket_name)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Now you can open your Google Cloud Platform console and check for your bucket with your files","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Read from GCS\nThese are some helper functions to deserialize and read data from TF Records","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"BATCH_SIZE = 128\ndef read_labeled_tfrecord(raw_example):\n    LABELED_TFREC_FORMAT = {\n        \"f0\": tf.io.FixedLenFeature([SEQUENCE_LENGTH], tf.int64),\n        \"f1\": tf.io.FixedLenFeature([SEQUENCE_LENGTH], tf.int64), \n        \"f2\": tf.io.FixedLenFeature([SEQUENCE_LENGTH], tf.int64), \n        \"y\":  tf.io.FixedLenFeature([], tf.int64),  \n    }\n\n    example = tf.io.parse_single_example(raw_example, LABELED_TFREC_FORMAT)\n    input_ids = tf.cast(example['f0'], tf.int64)\n    input_mask = tf.cast(example['f1'], tf.int64)\n    segment_ids =  tf.cast(example['f2'], tf.int64)\n    y = tf.cast(example['y'], tf.int64)\n    return (input_ids,input_mask,segment_ids) , y # returns a dataset of (image, label) pairs\n\ndef load_dataset(filenames, ordered = False):\n    # Read from TFRecords. For optimal performance, reading from multiple files at once and\n    # Diregarding data order. Order does not matter since we will be shuffling the data anyway\n    \n    ignore_order = tf.data.Options()\n    if not ordered:\n        ignore_order.experimental_deterministic = False # disable order, increase speed\n        \n    gs_path = tf.io.gfile.glob(filenames)\n    dataset = tf.data.TFRecordDataset(gs_path, num_parallel_reads = AUTO) # automatically interleaves reads from multiple files\n    dataset = dataset.with_options(ignore_order) # use data as soon as it streams in, rather than in its original order\n    dataset = dataset.map(read_labeled_tfrecord, num_parallel_calls = AUTO) # returns a dataset of (image, label) pairs if labeled = True or (image, id) pair if labeld = False\n    return dataset\n\ndef get_training_dataset(dataset, validation=False):\n    if not validation:\n        return dataset.repeat().shuffle(2048).batch(BATCH_SIZE).prefetch(AUTO)\n    else:\n        return dataset.batch(BATCH_SIZE).cache().prefetch(AUTO)\n    \n# Usage would be:\n# get_training_dataset( load_dataset( filenames = 'gs://your_bucket_name/train*.tfrec'))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":1}