{"cells":[{"metadata":{"_uuid":"8bf878b1-94a1-444f-b6e6-4e2e95e930eb","_cell_guid":"d015aaad-2e3b-41ed-8344-93e1ea021493","trusted":true},"cell_type":"code","source":"\nimport numpy as np \nimport pandas as pd \n\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"84332d3e-56f1-4cbb-a99f-0eb3aa9da58b","_cell_guid":"1b368c6a-f654-4c10-a1b1-e25293929ab3","trusted":true},"cell_type":"code","source":"\nimport tensorflow as tf\nprint(\"Tensorflow version \" + tf.__version__)\n\ntry:\n  tpu = tf.distribute.cluster_resolver.TPUClusterResolver()  # TPU detection\n  print('Running on TPU ', tpu.cluster_spec().as_dict()['worker'])\nexcept ValueError:\n  raise BaseException('ERROR: Not connected to a TPU runtime; please see the previous cell in this notebook for instructions!')\n\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\ntpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d46dd289-b10b-484f-806c-4294ab081299","_cell_guid":"bf121066-7ed7-4b10-9cb9-96374cce06f9","trusted":true},"cell_type":"code","source":"import pandas as pd\nimport numpy as np\nimport matplotlib.pyplot as plt\nimport re","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b64df15b-436a-4c69-8981-fdfdf5aa00f9","_cell_guid":"91f7ae1c-9fb0-4d59-a248-56787e9f45c5","trusted":true},"cell_type":"code","source":"train = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"dfb952b1-40fb-4ed8-ae67-349b0cb50a76","_cell_guid":"744a902a-c192-4055-bed3-f1f480e1beee","trusted":true},"cell_type":"code","source":"train.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d48288b6-94de-417f-8920-8a92c5f7ac1d","_cell_guid":"96cdd5a4-c467-4a16-b93b-43bebf5a653a","trusted":true},"cell_type":"code","source":"train.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b586d419-cf63-4638-84b3-577cb43df507","_cell_guid":"544649e4-c587-474f-b099-d1bc117e309e","trusted":true},"cell_type":"code","source":"#Data cleaning\n\nimport re\n\ndef text_process(text):\n    '''Make text lowercase, remove text in square brackets,remove all single character, Substituting multiple spaces with single space,removing all special characters,remove links,remove punctuation\n    and remove words containing numbers.'''\n    text = text.lower()\n    text = re.sub('\\[.*?\\]#', '', text)\n    #text = re.sub(r'\\W', ' ', str(X[text]))\n    text = re.sub(r'\\s+[a-zA-Z]\\s+', ' ', text)\n    text = re.sub('https?://\\S+|www\\.\\S+', '', text)\n    text = re.sub('<.*?>+', '', text)\n    text = re.sub(r'\\s+', ' ', text, flags=re.I)\n    #text = re.sub('[%s]' % re.escape(str.punctuation), '', text)\n    text = re.sub('\\n', '', text)\n    text = re.sub('\\w*\\d\\w*', '', text)\n    text = re.sub('©', '', text)\n    text = re.sub('@', '', text)\n    text = re.sub('#', '', text)\n    text = re.sub('‰ûò', '', text)\n    text = re.sub('!', '', text)\n    text = re.sub('&', '', text)\n \n    #text = re.sub('?', '', text)\n    return text","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1ada6f31-3040-43d4-afc9-e09188730b70","_cell_guid":"7cc4bd5a-7d31-434c-9e4b-f77d4150eb47","trusted":true},"cell_type":"code","source":"train['comment_text'] = train['comment_text'].apply(lambda x: text_process(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"ae9a2e33-f20b-4d3c-9969-0423501568b3","_cell_guid":"8b1c8c4d-8574-4ab8-a9b6-767516cd01c5","trusted":true},"cell_type":"code","source":"train = train.rename(columns={\"comment_text\": \"text\"})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c59e6c6d-5dd9-4f9a-8a22-5d7298cf31e1","_cell_guid":"fc02f735-fc37-417b-8df7-29b133b65196","trusted":true},"cell_type":"code","source":"#Validation dataset\n\nvalid = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/validation-processed-seqlen128.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cce049c7-d001-4b23-b99c-8fb1873cbc77","_cell_guid":"ff5b2a85-4561-48c6-9950-d2fe0d17ee3a","trusted":true},"cell_type":"code","source":"valid.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"40432a5b-db6b-41b1-a772-a12928fadeb8","_cell_guid":"0c92b85c-5710-4d66-9c06-d3d1fcbf6b58","trusted":true},"cell_type":"code","source":"# Applying the cleaning function \nvalid['comment_text'] = valid['comment_text'].apply(lambda x: text_process(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d1703afe-d07e-453b-b252-3b707c61fade","_cell_guid":"0494bfe2-6ee3-4bad-b7a9-2531f77ab7f8","trusted":true},"cell_type":"code","source":"valid = valid.rename(columns={\"comment_text\": \"text\"})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a7bbf82f-007c-42c6-b49f-06d8034efc6c","_cell_guid":"5b6d59a2-a192-49ee-94c4-d97c4283285e","trusted":true},"cell_type":"code","source":"#Load test dataset\n\ntest = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/test-processed-seqlen128.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"44fc3c25-d086-4ba8-8a90-63deae69f059","_cell_guid":"63008b6b-3909-4de6-91a4-2d8188eeba89","trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"474b4c3a-fc93-4d12-a251-5781f524182c","_cell_guid":"b5ec3e3d-7955-4473-b77a-0e6a8f2af856","trusted":true},"cell_type":"code","source":"# Applying the cleaning function \ntest['comment_text'] = test['comment_text'].apply(lambda x: text_process(x))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"fe5742e8-2be8-4825-b3e8-20f34da8d099","_cell_guid":"49a93201-efbb-478f-b2b6-349de51690c1","trusted":true},"cell_type":"code","source":"test = test.rename(columns={\"comment_text\": \"text\"})","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a0250923-b709-45c0-8cfd-5c4296b4ec0d","_cell_guid":"bb68be22-cf74-47ef-848b-71a0cc470c3c","trusted":true},"cell_type":"code","source":"!pip install transformers","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f98b8bc-784e-40e0-8731-d9cfed86ec37","_cell_guid":"a93340a2-86ee-4910-9734-5fe967918a48","trusted":true},"cell_type":"code","source":"#Bert Model\n\nimport tensorflow\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam, Nadam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0b07e268-eda6-43e5-93c7-605dbdde17c2","_cell_guid":"68bfc31e-59e1-478d-9f63-cc240be8d666","trusted":true},"cell_type":"code","source":"#Function for encoding the word/comment to integer or vector\n\ndef regular_encode(texts, tokenizer, maxlen=512):\n  enc_di = tokenizer.batch_encode_plus(\n      texts,\n      return_attention_masks=False,\n      return_token_type_ids=False,\n      pad_to_max_length=True,\n      max_length=maxlen\n  )\n\n  return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1aef3b65-c91e-49bb-8f7d-f07714971143","_cell_guid":"cd5ef120-6b6b-4669-93b2-3350b3a45cd6","trusted":true},"cell_type":"code","source":"#BUILD MODEL FUNCTION\n\ndef build_model(transformer, max_len=512):\n  input_word_ids = Input(shape=max_len, dtype=tf.int32, name='input_word_ids')\n  sequence_output=transformer(input_word_ids)[0]\n  cls_token=sequence_output[:, 0, :]\n  out = Dense(1, activation='sigmoid')(cls_token)\n\n  model = Model(inputs=input_word_ids, outputs=out)\n  model.compile(Adam(lr=1e-5), loss='binary_crossentropy',metrics=[tf.keras.metrics.AUC()])\n\n  return model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a5d24c4b-43d0-4c5a-9eb4-873007bd3c04","_cell_guid":"80678f1a-943e-4385-8047-1975673d2991","trusted":true},"cell_type":"code","source":"EPOCHS = 10\nBATCH_SIZE = 16 * tpu_strategy.num_replicas_in_sync \nMAX_LEN = 512  \nMODEL = 'bert-base-multilingual-cased'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e3054968-7e98-4e6b-8d6e-8ee6cf85e1b1","_cell_guid":"59e0d854-4d62-4b8f-a850-af9e25bb497f","trusted":true},"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9a257d38-6cc3-4473-951c-fd97e9c7e5b1","_cell_guid":"8579ca3a-b07d-4bfe-b980-2ef14289cae9","trusted":true},"cell_type":"code","source":"#Encode comments (text into vectors)\n\nx_train = regular_encode(train.text.values, tokenizer, maxlen=MAX_LEN)\n\nx_valid = regular_encode(valid.text.values, tokenizer, maxlen=MAX_LEN)\n\ny_train = train.toxic.values\n\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c7149d2b-6f90-4a5f-bd7d-3092e130dda9","_cell_guid":"ba862838-df78-47d4-a406-8bf72c4361c8","trusted":true},"cell_type":"code","source":"x_test = regular_encode(test.text.values, tokenizer, maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"bdc13b86-15f2-47a0-8fc3-06d62e15cbb2","_cell_guid":"4a6d5b3a-56d9-420c-817a-8b10d86e1744","trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a9fd58bd-29ad-44ea-82c6-c6ee03336df3","_cell_guid":"39afc03a-d24b-45c0-9703-12a2d7b0584b","trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)    \n    .batch(BATCH_SIZE)\n    .prefetch(AUTO) \n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7fdcb358-cb8e-4b34-b6fb-3b1babb4b27a","_cell_guid":"5e479c52-5615-4946-9afc-c74bd4384935","trusted":true},"cell_type":"code","source":"valid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))  #no shuffle or repeat\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"34432074-0613-4fa4-9c5e-42bb8c96c91f","_cell_guid":"beded954-3cd4-441e-8195-d8ac58539ac6","trusted":true},"cell_type":"code","source":"test_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test) \n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"79ef14c1-7652-4fb3-ac2a-2fb142702a4d","_cell_guid":"1a6cd60b-7849-4801-8035-80830e24e5b7","trusted":true},"cell_type":"code","source":"#Model","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"de391805-5008-4611-b90f-19c3b103802b","_cell_guid":"76f0d182-9b8f-4a1a-b172-c0c37f6c4575","trusted":true},"cell_type":"code","source":"MODEL = 'bert-base-multilingual-cased'","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cff5fe60-0fc5-4267-b5c2-b24c475359b2","_cell_guid":"9b06a8e1-b45b-4921-a67d-b01d66fa85f3","trusted":true},"cell_type":"code","source":"#Build model from BERT pretrained model and the build model function\n\nwith tpu_strategy.scope():   #enables to use the TPU while training the model\n  transformer_layer = TFAutoModel.from_pretrained(MODEL)\n  model = build_model(transformer_layer, max_len = MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"a790d23f-e46b-41d6-b7bc-1551a19b6c9a","_cell_guid":"37eabbb9-da92-4eb5-bc79-8a30e2c0199e","trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"89b128cd-78e3-4f15-8bcb-adb8b12847f4","_cell_guid":"9dc92e4d-1d74-4427-b878-ecf848bf865f","trusted":true},"cell_type":"code","source":"!pip install h5py","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f0ef2940-3c26-45ee-9268-da41cdfb8470","_cell_guid":"7ca92640-994d-4dae-8e8a-4a47cf15318a","trusted":true},"cell_type":"code","source":"#Call backs\n\nEPOCHS = 2\n\nfrom tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, LearningRateScheduler\n\n#stop = EarlyStopping(monitor='val_auc', mode='max', min_delta=0.01, patience=1, verbose=1)\n\n#rlrop = ReduceLROnPlateau(monitor='val_auc', mode='auto', min_delta=0.01, factor=0.2, patience=1) \n\n#filepath = '/kaggle/working/bert_model_tpu_v2.hdf5'   # Saved model checkpoint file path\n\n#checkpoint = ModelCheckpoint(filepath, monitor='val_auc', verbose=1, save_best_only=True, mode='max', save_freq='epoch')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"96e87836-a7a1-4aee-ac65-aba875301fc5","_cell_guid":"5025409d-366a-497c-8d4d-5bc9521f85b3","trusted":true},"cell_type":"code","source":"start_lr = 0.00001\nmin_lr = 0.00001\nmax_lr = 0.00005 * tpu_strategy.num_replicas_in_sync\n\nrampup_epochs = 5\nsustain_epochs = 0\nexp_decay=0.8\n\ndef lrfn(epoch):\n  if epoch < rampup_epochs:\n    return (max_lr - start_lr)/rampup_epochs * epoch + start_lr\n  elif epoch < rampup_epochs + sustain_epochs:\n    return max_lr\n  else:\n    return (max_lr - min_lr) * exp_decay**(epoch-rampup_epochs-sustain_epochs) + min_lr\n\nlr_callback = LearningRateScheduler(lambda epoch: lrfn(epoch), verbose=True)\n\nrang = np.arange(EPOCHS)\ny = [lrfn(x) for x in rang]\nplt.plot(rang, y)\n\nprint('Learning rate per epoch:')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"b4e77354-0679-4a73-bb57-18cc23552e75","_cell_guid":"97f0aa47-7292-4f40-b2b9-b7911c6d46c1","trusted":true},"cell_type":"code","source":"#Model training\n\nn_steps = x_train.shape[0] // BATCH_SIZE\n\nmodel.fit(train_dataset, steps_per_epoch=n_steps, validation_data=valid_dataset, epochs=2, callbacks=[lr_callback])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"681db966-c3aa-48ea-80f1-816f766e5bd3","_cell_guid":"b25a2412-1144-4bea-8ffb-19c40e1114c2","trusted":true},"cell_type":"code","source":"n_steps = x_valid.shape[0] // BATCH_SIZE\nmodel.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=2\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c93ed53b-3c4d-49d9-80be-738a49eb7793","_cell_guid":"08d9dd4e-60b8-405b-9096-a4a673f9fb66","trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7ab75cac-1ad5-42df-9604-fb9536dd8922","_cell_guid":"367d7bf5-84ea-4fc6-843a-33bbf9d0e2ce","trusted":true},"cell_type":"code","source":"sub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}