{"cells":[{"metadata":{"_cell_guid":"d015aaad-2e3b-41ed-8344-93e1ea021493","_uuid":"8bf878b1-94a1-444f-b6e6-4e2e95e930eb","trusted":true},"cell_type":"code","source":"\nimport numpy as np \nimport pandas as pd \n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1b368c6a-f654-4c10-a1b1-e25293929ab3","_uuid":"84332d3e-56f1-4cbb-a99f-0eb3aa9da58b","trusted":true},"cell_type":"code","source":"\nimport tensorflow as tf\nprint(\"Tensorflow version \" + tf.__version__)\n\ntry:\n  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n  print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nexcept ValueError:\n  raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bf121066-7ed7-4b10-9cb9-96374cce06f9","_uuid":"d46dd289-b10b-484f-806c-4294ab081299","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport re","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"91f7ae1c-9fb0-4d59-a248-56787e9f45c5","_uuid":"b64df15b-436a-4c69-8981-fdfdf5aa00f9","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"744a902a-c192-4055-bed3-f1f480e1beee","_uuid":"dfb952b1-40fb-4ed8-ae67-349b0cb50a76","trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"96cdd5a4-c467-4a16-b93b-43bebf5a653a","_uuid":"d48288b6-94de-417f-8920-8a92c5f7ac1d","trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"544649e4-c587-474f-b099-d1bc117e309e","_uuid":"b586d419-cf63-4638-84b3-577cb43df507","trusted":true},"cell_type":"code","source":"#Data cleaning\n\nimport re\n\ndef text_process(text):\n    '''Make text lowercase, remove text in square brackets,remove all single character, Substituting multiple spaces with single space,removing all special characters,remove links,remove punctuation\n    and remove words containing numbers.'''\n    text = text.lower()\n    text = re.sub('\\[.*?\\]#', '', text)\n    #text = re.sub(r'\\W', ' ', str(X[text]))\n    text = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', text)\n    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n    text = re.sub('<.*?>+', '', text)\n    text = re.sub(r'\\s+', ' ', text, flags=re.I)\n    #text = re.sub('[%s]' % re.escape(str.punctuation), '', text)\n    text = re.sub('\\n', '', text)\n    text = re.sub('\\w*\\d\\w*', '', text)\n    text = re.sub('©', '', text)\n    text = re.sub('@', '', text)\n    text = re.sub('#', '', text)\n    text = re.sub('‰ûò', '', text)\n    text = re.sub('!', '', text)\n    text = re.sub('&', '', text)\n \n    #text = re.sub('?', '', text)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7cc4bd5a-7d31-434c-9e4b-f77d4150eb47","_uuid":"1ada6f31-3040-43d4-afc9-e09188730b70","trusted":true},"cell_type":"code","source":"train['comment_text'] = train['comment_text'].apply(lambda x: text_process(x))","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8b1c8c4d-8574-4ab8-a9b6-767516cd01c5","_uuid":"ae9a2e33-f20b-4d3c-9969-0423501568b3","trusted":true},"cell_type":"code","source":"train = train.rename(columns={\"comment_text\": \"text\"})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"bb68be22-cf74-47ef-848b-71a0cc470c3c","_uuid":"a0250923-b709-45c0-8cfd-5c4296b4ec0d","trusted":true},"cell_type":"code","source":"!pip install transformers","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"a93340a2-86ee-4910-9734-5fe967918a48","_uuid":"7f98b8bc-784e-40e0-8731-d9cfed86ec37","trusted":true},"cell_type":"code","source":"#DistilBERT Model\n\nimport tensorflow\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam, Nadam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"68bfc31e-59e1-478d-9f63-cc240be8d666","_uuid":"0b07e268-eda6-43e5-93c7-605dbdde17c2","trusted":true},"cell_type":"code","source":"#Function for encoding the word/comment to integer or vector\n\ndef regular_encode(texts, tokenizer, maxlen=512):\n  enc_di = tokenizer.batch_encode_plus(\n      texts,\n      return_attention_masks=False,\n      return_token_type_ids=False,\n      pad_to_max_length=True,\n      max_length=maxlen\n  )\n\n  return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"cd5ef120-6b6b-4669-93b2-3350b3a45cd6","_uuid":"1aef3b65-c91e-49bb-8f7d-f07714971143","trusted":true},"cell_type":"code","source":"#BUILD MODEL FUNCTION\n\ndef build_model(transformer, max_len=512):\n  input_word_ids = Input(shape=max_len, dtype=tf.int32, name='input_word_ids')\n  sequence_output=transformer(input_word_ids)[0]\n  cls_token=sequence_output[:, 0, :]\n  out = Dense(1, activation='sigmoid')(cls_token)\n\n  model = Model(inputs=input_word_ids, outputs=out)\n  model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC()])\n\n  return model","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"80678f1a-943e-4385-8047-1975673d2991","_uuid":"a5d24c4b-43d0-4c5a-9eb4-873007bd3c04","trusted":true},"cell_type":"code","source":"EPOCHS = 10\nBATCH_SIZE = 16 * tpu_strategy.num_replicas_in_sync \nMAX_LEN = 512  \nMODEL = 'distilbert-base-cased'","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"59e0d854-4d62-4b8f-a850-af9e25bb497f","_uuid":"e3054968-7e98-4e6b-8d6e-8ee6cf85e1b1","trusted":true},"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Train, valid split\n\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train, valid = train_test_split(train, test_size=0.20, stratify=train['toxic'], random_state=42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape, valid.shape","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"8579ca3a-b07d-4bfe-b980-2ef14289cae9","_uuid":"9a257d38-6cc3-4473-951c-fd97e9c7e5b1","trusted":true},"cell_type":"code","source":"#Encode comments (text into vectors)\n\nx_train = regular_encode(train.text.values, tokenizer, maxlen=MAX_LEN)\n\nx_valid = regular_encode(valid.text.values, tokenizer, maxlen=MAX_LEN)\n\ny_train = train.toxic.values\n\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"4a6d5b3a-56d9-420c-817a-8b10d86e1744","_uuid":"bdc13b86-15f2-47a0-8fc3-06d62e15cbb2","trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"39afc03a-d24b-45c0-9703-12a2d7b0584b","_uuid":"a9fd58bd-29ad-44ea-82c6-c6ee03336df3","trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)    \n    .batch(BATCH_SIZE)\n    .prefetch(AUTO) \n)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5e479c52-5615-4946-9afc-c74bd4384935","_uuid":"7fdcb358-cb8e-4b34-b6fb-3b1babb4b27a","trusted":true},"cell_type":"code","source":"valid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))  #no shuffle or repeat\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"1a6cd60b-7849-4801-8035-80830e24e5b7","_uuid":"79ef14c1-7652-4fb3-ac2a-2fb142702a4d","trusted":true},"cell_type":"code","source":"#Model","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9b06a8e1-b45b-4921-a67d-b01d66fa85f3","_uuid":"cff5fe60-0fc5-4267-b5c2-b24c475359b2","trusted":true},"cell_type":"code","source":"%%time\nwith tpu_strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer, max_len=MAX_LEN)\n","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"37eabbb9-da92-4eb5-bc79-8a30e2c0199e","_uuid":"a790d23f-e46b-41d6-b7bc-1551a19b6c9a","trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"9dc92e4d-1d74-4427-b878-ecf848bf865f","_uuid":"89b128cd-78e3-4f15-8bcb-adb8b12847f4","trusted":true},"cell_type":"code","source":"!pip install h5py","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"7ca92640-994d-4dae-8e8a-4a47cf15318a","_uuid":"f0ef2940-3c26-45ee-9268-da41cdfb8470","trusted":true},"cell_type":"code","source":"#Call backs\n\nEPOCHS = 2\n\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, LearningRateScheduler\n\n#stop = EarlyStopping(monitor='val_auc', mode='max', min_delta=0.01, patience=1, verbose=1)\n\n#rlrop = ReduceLROnPlateau(monitor='val_auc', mode='auto', min_delta=0.01, factor=0.2, patience=1) \n\n#filepath = '/kaggle/working/bert_model_tpu_v2.hdf5'   # Saved model checkpoint file path\n\n#checkpoint = ModelCheckpoint(filepath, monitor='val_auc', verbose=1, save_best_only=True, mode='max', save_freq='epoch')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"5025409d-366a-497c-8d4d-5bc9521f85b3","_uuid":"96e87836-a7a1-4aee-ac65-aba875301fc5","trusted":true},"cell_type":"code","source":"start_lr = 0.00001\nmin_lr = 0.00001\nmax_lr = 0.00005 * tpu_strategy.num_replicas_in_sync\n\nrampup_epochs = 5\nsustain_epochs = 0\nexp_decay=0.8\n\ndef lrfn(epoch):\n  if epoch < rampup_epochs:\n    return (max_lr - start_lr)/rampup_epochs * epoch + start_lr\n  elif epoch < rampup_epochs + sustain_epochs:\n    return max_lr\n  else:\n    return (max_lr - min_lr) * exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr\n\nlr_callback = LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=True)\n\nrang = np.arange(EPOCHS)\ny = [lrfn(x) for x in rang]\nplt.plot(rang, y)\n\nprint('Learning rate per epoch:')","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"97f0aa47-7292-4f40-b2b9-b7911c6d46c1","_uuid":"b4e77354-0679-4a73-bb57-18cc23552e75","trusted":true},"cell_type":"code","source":"#Model training\n\nn_steps = x_train.shape[0] // BATCH_SIZE\n\nmodel.fit(train_dataset, steps_per_epoch=n_steps, validation_data=valid_dataset, epochs=2, callbacks=[lr_callback])","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"b25a2412-1144-4bea-8ffb-19c40e1114c2","_uuid":"681db966-c3aa-48ea-80f1-816f766e5bd3","trusted":true},"cell_type":"code","source":"#Fit model on validation dataset for two more epochs\n\n#Validation AUC = 99.24% using Distil-BERT Model\n\nn_steps = x_valid.shape[0] // BATCH_SIZE\nmodel.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=2\n)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}