{"cells":[{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"from kaggle_datasets import KaggleDatasets\nimport numpy as np\nimport pandas as pd\nimport os\nimport tensorflow as tf\nimport re\nimport transformers\nimport tqdm\nimport gc\nimport time\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nfrom tokenizers import BertWordPieceTokenizer\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(tf.version.VERSION)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**LOAD DATASET**"},{"metadata":{"trusted":true},"cell_type":"code","source":"val_data = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest_data = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\ntrain_data1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\n\ntrain_data2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\ntrain_data2.toxic = train_data2.toxic.round().astype(int)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data= pd.concat([\n    train_data1[['comment_text', 'toxic']],\n    train_data2[['comment_text', 'toxic']].query('toxic==1'),\n    train_data2[['comment_text', 'toxic']].query('toxic==0').sample(n=150000, random_state=0)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.shape,val_data.shape,test_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SEQ_LENGTH = 512","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**CLEANING INPUT TEXTS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean(text):\n    # repalce na values\n    text = text.fillna(\"fillna\").str.lower()\n    #replace newline characters with space\n    text = text.map(lambda x: re.sub('\\\\n',' ', str(x)))\n    # remove extra characters\n    text = text.map(lambda x: re.sub('\\[\\[User.*', '', str(x)))\n    text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_data[\"comment_text\"] = clean(val_data[\"comment_text\"])\ntest_data[\"content\"] = clean(test_data[\"content\"])\ntrain_data[\"comment_text\"] = clean(train_data[\"comment_text\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**BERT TOKENIZER**"},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**PREPROCESSING**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_bert_input(tokenizer, docs, max_seq_len):\n    all_input_ids, all_mask_ids = [], []\n    for doc in tqdm.tqdm(docs, desc=\"Converting docs to features\"):\n        tokens = tokenizer.tokenize(doc)\n        if len(tokens) > max_seq_len - 2:\n            tokens = tokens[0: (max_seq_len-2)]\n        tokens = ['[CLS]']+tokens+['[SEP]']\n        ids = tokenizer.convert_tokens_to_ids(tokens)\n        masks = [1]*len(ids)\n        while len(ids) < max_seq_len:\n            ids.append(0)\n            masks.append(0)\n        all_input_ids.append(ids)\n        all_mask_ids.append(masks)\n    \n    encoded = np.array([all_input_ids, all_mask_ids])\n    return encoded","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_comments = train_data.comment_text.astype(str).values\nval_comments = val_data.comment_text.astype(str).values\ntest_comments = test_data.content.astype(str).values\ny_valid = val_data.toxic.values\ny_train = train_data.toxic.values\n\ntrain_comments.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feature_ids, train_feature_masks = create_bert_input(tokenizer, train_comments, max_seq_len=MAX_SEQ_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_feature_ids, val_feature_masks = create_bert_input(tokenizer, val_comments, max_seq_len=MAX_SEQ_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_feature_ids.shape, train_feature_masks.shape, y_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_feature_ids.shape, val_feature_masks.shape, y_valid.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**TPU CONFIGURATION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"## TPU configuration\n\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\n\nEPOCHS = 2\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**BUILDING DATASETS OBJECTS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_ds = (\n    tf.data.Dataset\n    .from_tensor_slices(((train_feature_ids, train_feature_masks), y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_ds = (\n    tf.data.Dataset\n    .from_tensor_slices(((val_feature_ids, val_feature_masks), y_valid))\n    .repeat()\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**MODEL**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_training_model():\n    inp_id = tf.keras.layers.Input(shape=(MAX_SEQ_LENGTH,), dtype=tf.int64, name=\"bert_input_ids\")\n    inp_mask = tf.keras.layers.Input(shape=(MAX_SEQ_LENGTH,), dtype=tf.int64, name=\"bert_input_masks\")\n    inputs = [inp_id, inp_mask]\n    \n    hidden_state = transformers.TFDistilBertModel.from_pretrained('distilbert-base-multilingual-cased')(inputs)[0]\n    pooled_output = hidden_state[:, 0]\n    dense1 = tf.keras.layers.Dense(128, activation='relu')(pooled_output)\n    output = tf.keras.layers.Dense(1, activation='sigmoid')(dense1)\n    model = tf.keras.Model(inputs=inputs, outputs=output)\n    model.compile(optimizer=tf.optimizers.Adam(learning_rate=2e-5, \n                                            epsilon=1e-08), \n                loss='binary_crossentropy', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**TRAINING MODEL**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model\nimport time\n\nstart = time.time()\n\n# Compile the model with TPU Strategy\nwith strategy.scope():\n    model = get_training_model()\n    \nmodel.fit(train_ds, \n          steps_per_epoch=train_data.shape[0] // BATCH_SIZE,\n          validation_data=valid_ds,\n          validation_steps=val_data.shape[0] // BATCH_SIZE,\n          epochs=EPOCHS,\n          verbose=1)\nend = time.time() - start\nprint(\"Time taken \",end)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**PREDICTION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"test_feature_ids, test_feature_masks = create_bert_input(tokenizer, test_comments, max_seq_len=MAX_SEQ_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sub['toxic'] = model.predict(test_ds, verbose=1)\nsub['toxic'] = model.predict([test_feature_ids, test_feature_masks], verbose=1)\n#sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**SUBMISSION**"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}