{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Importing required Libraries","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install ipython-autotime\n!pip install --upgrade transformers\n%matplotlib inline\n%load_ext autotime","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import os,re\nimport unicodedata\nimport gc\nimport time\nimport numpy as np \nimport pandas as pd\n\nimport matplotlib.pyplot as plt\n\nimport nltk\nfrom nltk.corpus import stopwords\n\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input, BatchNormalization, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tensorflow.keras.losses import BinaryCrossentropy\n\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom transformers import T5Tokenizer, TFT5Model\nfrom transformers import TFRobertaModel, RobertaTokenizerFast, RobertaConfig\nfrom tokenizers import BertWordPieceTokenizer\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\n\nfrom tqdm.notebook import tqdm\nfrom numba import jit, cuda ","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU Configuration","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Reading Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# the data paths\ndata_path = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'\ntranslated_data_path = '/kaggle/input/jigsaw-multilingual-toxic-test-translated/'\n\n# loading all the train datasets\n\ntrain_data1 = pd.read_csv(data_path + 'jigsaw-toxic-comment-train.csv')\n# train_data2 = pd.read_csv(data_path + 'jigsaw-toxic-comment-train-processed-seqlen128.csv')\ntrain_data3 = pd.read_csv(data_path + 'jigsaw-unintended-bias-train.csv')\n# train_data4 = pd.read_csv(data_path + 'jigsaw-unintended-bias-train-processed-seqlen128.csv')\n\n# loading all the validation and test datasets\n\n# validation_data1 = pd.read_csv(data_path + 'validation.csv')\n# validation_data2 = pd.read_csv(data_path + 'validation-processed-seqlen128.csv')\nvalid_translated = pd.read_csv(translated_data_path + 'jigsaw_miltilingual_valid_translated.csv')\n\n# test_data1 = pd.read_csv(data_path + 'test.csv')\n# test_data2 = pd.read_csv(data_path + 'test-processed-seqlen128.csv')\ntest_translated = pd.read_csv(translated_data_path + 'jigsaw_miltilingual_test_translated.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Exploratory Data Analysis\n\n*Link to the notebook:*\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Data cleaning Functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Stopword list\npattern = re.compile(r'\\b('+r'|'.join(stopwords.words('english'))+r')\\b\\s*')\n\n# @cuda.jit(device=True)\ndef unicode_to_ascii(s):\n  return ''.join(c for c in unicodedata.normalize('NFD', s)\n      if unicodedata.category(c) != 'Mn')\n\n# @tf.function()\ndef clean_text(text):\n    text = unicode_to_ascii(text.lower().strip())\n    \n    #replacing email addresses with blank space\n    text = re.sub(r\"[a-zA-Z0-9_\\-\\.]+@[a-zA-Z0-9_\\-\\.]+\\.[a-zA-Z]{2,5}\",\" \",text)\n    \n    #replacing urls with blank space\n    text = re.sub(r\"\\bhttp:\\/\\/([^\\/]*)\\/([^\\s]*)|https:\\/\\/([^\\/]*)\\/([^\\s]*)\",\" \",text)\n    \n    # creating a space between a word and the punctuation following it\n    text = re.sub(r\"([?.!,¿])\", r\" \\1 \", text)\n    text = re.sub(r'[\" \"]+', \" \", text)\n    \n    # replacing all the stopwords\n    text = pattern.sub('',text)\n    \n    # removes all the punctuations\n    text = re.sub(r\"[^a-zA-Z]+\", \" \", text)\n    \n    text = text.strip()\n\n    # adding a start and an end token to the sentence so that the model know when to start and stop predicting.\n#     text = '<start> ' + text + ' <end>'\n    \n    return text\n\nclean_text_vect = np.vectorize(clean_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def chunk_clean(array,chunk_size=256):\n    cleaned_array = []\n    \n    for i in tqdm(range(0, len(array), chunk_size)):\n        text_chunk = clean_text_vect(array[i:i+chunk_size])\n        cleaned_array.extend(text_chunk)\n\n    return np.array(cleaned_array)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Tokenizing and Encoding Functions","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    \n    tokenizer.pad_token = tokenizer.pad_token\n    tokenizer.unk_token = tokenizer.unk_token\n    tokenizer.eos_token = tokenizer.eos_token\n    \n    enc_di = tokenizer.batch_encode_plus(\n        list(texts), \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen,\n        add_special_tokens=True\n    )\n    \n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def chunk_encode(texts,tokenizer,maxlen=512,chunk_size=256):\n    all_enc=[]\n    for i in tqdm(range(0,len(texts),chunk_size)):\n        enc = list(regular_encode(texts[i:i+chunk_size],tokenizer,maxlen=maxlen))\n        all_enc.extend(enc)\n        \n    return np.array(all_enc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n\n    for i in tqdm(range(0, len(texts), chunk_size)):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n\n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Defining the Tokenizer","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Pre-trained models","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL = 'google/electra-large-generator'\nMODEL2 = 'google/electra-large-discriminator'\nMODEL3 = 'gpt2-medium'\nMODEL4 = 'roberta-large'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Initializing the Tokenizers","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = RobertaTokenizerFast.from_pretrained(MODEL4)\nprint(tokenizer.save_pretrained('.'))\nprint(tokenizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# bert_tokenizer = transformers.BertTokenizerFast.from_pretrained('bert-large-uncased')\n# bert_tokenizer.save_pretrained('.')\n# fast_tokenizer = BertWordPieceTokenizer('./vocab.json', lowercase=False)\n# fast_tokenizer","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n# Data Preparation","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Data splitting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data3.toxic = train_data3.toxic.round().astype(int)\nvalid_translated['comment_text'] = valid_translated['translated']\n\ndata = pd.concat([\n    train_data1[['comment_text', 'toxic']],\n    train_data3[['comment_text', 'toxic']].query('toxic==1'),\n    train_data3[['comment_text', 'toxic']].query('toxic==0').sample(n=200000, random_state=0),\n    valid_translated[['comment_text','toxic']]\n])\ndata.toxic = data.toxic.round().astype(int)\ndata.drop_duplicates(inplace=True)\n\nfinal_test_data = test_translated.translated.values\n\nprint('Number of toxic comments = ',list(data.toxic).count(1))\nprint('Number of non-toxic comments = ',list(data.toxic).count(0))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.random.seed(2048)\ntrain, valid, test = np.split(data.sample(frac=1), [int(.94*len(data)), int(.97*len(data))])\n\nprint(\"Train rows = \", train.shape[0])\nprint(\"validate rows = \", valid.shape[0])\nprint(\"Test rows = \", test.shape[0])\nprint(\"\\nFinal Test Data rows = \",len(final_test_data))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train_data1\ndel train_data3\ndel valid_translated\ndel data\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Cleaning & Tokenizing Input data and Preparing Labels","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_LEN = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = chunk_clean(train.comment_text.values)\nx_valid = chunk_clean(valid.comment_text.values)\nx_test = chunk_clean(test.comment_text.values)\n\nfinal_test_data = chunk_clean(final_test_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = chunk_encode(x_train, tokenizer, maxlen=MAX_LEN)\nx_valid = chunk_encode(x_valid, tokenizer, maxlen=MAX_LEN)\nx_test = chunk_encode(x_test, tokenizer, maxlen=MAX_LEN)\n\nfinal_test_data = chunk_encode(final_test_data, tokenizer, maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# x_train = fast_encode(x_train, fast_tokenizer, maxlen=MAX_LEN)\n# x_valid = fast_encode(x_valid, fast_tokenizer, maxlen=MAX_LEN)\n# x_test = fast_encode(x_test, fast_tokenizer, maxlen=MAX_LEN)\n\n# final_test_data = fast_encode(final_test_data, fast_tokenizer, maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = np.array(train.toxic.values)\ny_train.resize((len(y_train),1))\n\ny_valid = np.array(valid.toxic.values)\ny_valid.resize((len(y_valid),1))\n\ny_test = np.array(test.toxic.values)\ny_test.resize((len(y_test),1))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('New shape of comments and labels after TOKENIZATION and PROCESSING:-')\nprint('-'*50)\nprint('Data for Training and Evaluation:\\n')\nprint('x_train shape = ',x_train.shape)\nprint('x_valid shape = ',x_valid.shape)\nprint('x_test shape = ',x_test.shape)\nprint('-'*30)\nprint('Labels shapes:\\n')\nprint('y_train shape = ',y_train.shape)\nprint('y_valid shape = ',y_valid.shape)\nprint('y_test shape = ',y_test.shape)\nprint('-'*50)\nprint('The Final data for Predication:\\n')\nprint('final_test_data shape = ',final_test_data.shape)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Converting to Tensorflow dataset","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n#     .repeat()\n#     .shuffle(256)\n    .prefetch(AUTO)\n)\n\ntest = (\n    tf.data.Dataset\n    .from_tensor_slices((x_test,y_test))\n    .batch(BATCH_SIZE)\n    .cache()\n#     .shuffle(256)\n    .prefetch(AUTO)\n)\n\nfinal_test_data = (\n    tf.data.Dataset\n    .from_tensor_slices(final_test_data)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Loading Model into TPU ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Creating the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"lr_schedule = tf.keras.optimizers.schedules.ExponentialDecay(\n    initial_learning_rate=1e-5,\n    decay_steps=1000,\n    decay_rate=0.9)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    \n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n            \n    out = Dense(64,activation=tf.nn.swish)(cls_token)\n    out = Dense(16,activation=tf.nn.swish)(out)\n    out = Dense(1, activation='sigmoid')(out)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    \n    model.compile(Adam(lr=1e-5),\n                  loss='binary_crossentropy',\n                  metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"configs = {\n  \"architectures\": [\n    \"RobertaForMaskedLM\"\n  ],\n  \"attention_probs_dropout_prob\": 0.1,\n  \"bos_token_id\": 0,\n  \"eos_token_id\": 2,\n  \"hidden_act\": \"gelu\",\n  \"hidden_dropout_prob\": 0.2,\n  \"hidden_size\": 1024,\n  \"initializer_range\": 0.02,\n  \"intermediate_size\": 4096,\n  \"layer_norm_eps\": 1e-05,\n  \"max_position_embeddings\": 514,\n  \"model_type\": \"roberta\",\n  \"num_attention_heads\": 16,\n  \"num_hidden_layers\": 24,\n  \"pad_token_id\": 1,\n  \"type_vocab_size\": 1,\n  \"vocab_size\": 50265\n}\n\nconfiguration = RobertaConfig.from_dict(configs)\nconfiguration","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    transformer_layer = transformers.TFAutoModel.from_pretrained(MODEL4)\n#     transformer_layer = TFRobertaModel(configuration)\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training the Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"EPOCHS = 3","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Stage 1","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_train.shape[0] // BATCH_SIZE\ntrain_history = model.fit(\n    train,\n    steps_per_epoch=n_steps,\n    validation_data=valid,\n#     validation_steps=100,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"epochs_range = range(EPOCHS)\n\nplt.figure(figsize=(16, 5))\n\nplt.subplot(121)\nplt.plot(epochs_range,train_history.history['accuracy'], label='accuracy')\nplt.plot(epochs_range,train_history.history['val_accuracy'], label = 'val_accuracy')\nplt.ylim(0.75,1)\nplt.xlabel('Epoch')\nplt.ylabel('Accuracy')\nplt.legend(loc='center right')\n\nplt.subplot(122)\nplt.plot(epochs_range,train_history.history['loss'], label='loss')\nplt.plot(epochs_range,train_history.history['val_loss'], label = 'val_loss')\nplt.ylim(0.1,0.35)\nplt.xlabel('Epoch')\nplt.ylabel('Loss')\nplt.legend(loc='center right')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss,accuracy = model.evaluate(test,verbose=1)\nprint('Loss = ',loss*100,'%')\nprint('Accuracy = ',accuracy*100,'%')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Stage 2","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# n_steps = x_valid.shape[0] // BATCH_SIZE\n# train_history_2 = model.fit(\n#     valid_dataset.repeat(),\n#     steps_per_epoch=n_steps,\n#     epochs=EPOCHS*2\n# )","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Creating submission file","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub1 = pd.read_csv(data_path + 'sample_submission.csv')\nsub1['toxic'] = model.predict(final_test_data, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub1.to_csv('submission.csv', index=False)\nsub1.head(15)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# THE END","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}