{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import tensorflow as tf\nprint(tf.__version__)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\nimport os\n\ndata_path = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'\n\nTEST_PATH = os.path.join(data_path, \"test.csv\")\nVAL_PATH = os.path.join(data_path, \"validation.csv\")\nTRAIN_PATH = os.path.join(data_path, \"jigsaw-toxic-comment-train.csv\")\n\nval_data = pd.read_csv(VAL_PATH)\ntest_data = pd.read_csv(TEST_PATH)\ntrain_data = pd.read_csv(TRAIN_PATH)\n\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Clean Text\nimport re\n\ndef clean(text):\n    text = text.fillna(\"fillna\").str.lower()\n    #replace newline characters with space\n    text = text.map(lambda x: re.sub('\\\\n',' ', str(x)))\n    text = text.map(lambda x: re.sub('\\[\\[User.*', '', str(x)))\n    text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n    return text\n\nval_data[\"comment_text\"] = clean(val_data[\"comment_text\"])\ntest_data[\"content\"] = clean(test_data[\"content\"])\ntrain_data[\"comment_text\"] = clean(train_data[\"comment_text\"])                    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load DistilBERT tokenizer\nimport transformers\n\ntokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy\nimport tqdm\n\ndef create_bert_input(tokenizer, docs, max_seq_len):\n    all_input_ids, all_mask_ids = [], []\n    for doc in tqdm.tqdm(docs, desc=\"Converting docs to features\"):\n        tokens = tokenizer.tokenize(doc)\n        if len(tokens) > max_seq_len - 2:\n            tokens = tokens[0: (max_seq_len-2)]\n        tokens = ['[CLS]']+tokens+['[SEP]']\n        ids = tokenizer.convert_tokens_to_ids(tokens)\n        masks = [1]*len(ids)\n        while len(ids) < max_seq_len:\n            ids.append(0)\n            masks.append(0)\n        all_input_ids.append(ids)\n        all_mask_ids.append(masks)\n    \n    encoded = np.array([all_input_ids, all_mask_ids])\n    return encoded\n            ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_comments = train_data.comment_text.astype(str).values\nval_comments = val_data.comment_text.astype(str).values\ntest_comments = test_data.content.astype(str).values\n\ny_valid = val_data.toxic.values\ny_train = train_data.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import gc\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Encode the comments in train_set\nMAX_SEQ_LENGTH = 500\n\ntrain_feature_ids, train_feature_masks = create_bert_input(tokenizer, train_comments, max_seq_len=MAX_SEQ_LENGTH)\n\nval_feature_ids, val_feature_masks = create_bert_input(tokenizer, val_comments, max_seq_len=MAX_SEQ_LENGTH)\n\ntest_feature_ids, test_feature_masks = create_bert_input(tokenizer, test_comments, max_seq_len=MAX_SEQ_LENGTH)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Verify the shapes\nprint(train_feature_ids.shape, train_feature_masks.shape, y_train.shape)\nprint(val_feature_ids.shape, val_feature_masks.shape, y_valid.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Configure TPU\nfrom kaggle_datasets import KaggleDatasets\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\n\nEPOCHS = 1\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create TensorFlow datasets for better performance\ntrain_ds = (\n    tf.data.Dataset\n    .from_tensor_slices(((train_feature_ids, train_feature_masks), y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)\n    \nvalid_ds = (\n    tf.data.Dataset\n    .from_tensor_slices(((val_feature_ids, val_feature_masks), y_valid))\n    .repeat()\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)\n\ntest_ds = (\n    tf.data.Dataset\n    .from_tensor_slices((test_feature_ids, test_feature_masks))\n    .repeat()\n    .batch(BATCH_SIZE)\n    .prefetch(tf.data.experimental.AUTOTUNE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#Create training ready model\ndef get_training_model():\n    inp_id = tf.keras.layers.Input(shape=(MAX_SEQ_LENGTH,), dtype=tf.int64, name=\"bert_input_ids\")\n    inp_mask = tf.keras.layers.Input(shape=(MAX_SEQ_LENGTH,), dtype=tf.int64, name=\"bert_input_masks\")\n    inputs = [inp_id, inp_mask]\n    \n    hidden_state = transformers.TFDistilBertModel.from_pretrained('distilbert-base-multilingual-cased')(inputs)[0]\n    pooled_output = hidden_state[:, 0]\n    dense1 = tf.keras.layers.Dense(128, activation='relu')(pooled_output)\n    output = tf.keras.layers.Dense(1, activation='sigmoid')(dense1)\n    model = tf.keras.Model(inputs=inputs, outputs=output)\n    model.compile(optimizer=tf.optimizers.Adam(learning_rate=2e-5, \n                                            epsilon=1e-08), \n                loss='binary_crossentropy', metrics=['accuracy'])\n    return model\n    \n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Authorize wandb\nimport wandb\n\nwandb.login()\nfrom wandb.keras import WandbCallback\n\n# Initialize wandb\nwandb.init(project=\"jigsaw-toxic\", id=\"distilbert-tpu-kaggle-weighted\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create 32 random indices from the English only test comments\nRANDOM_INDICES = np.random.choice(test_comments.shape[0], 32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install -q googletrans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Demo examples of translations\nfrom googletrans import Translator\n\nsample_comment = test_comments[48649]\nprint(\"Original comment:\", sample_comment)\ntranslated_comment = Translator().translate(sample_comment)\nprint(\"\\n\")\nprint(\"Translated comment:\", translated_comment.text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create a sample prediction logger\n# A custom callback to view predictions on the above samples in real-time\nclass TextLogger(tf.keras.callbacks.Callback):\n    def __init__(self):\n        super(TextLogger, self).__init__()\n\n    def on_epoch_end(self, logs, epoch):\n        samples = []\n        for index in RANDOM_INDICES:\n            # Grab the comment and translate it\n            comment = test_comments[index]\n            translated_comment = Translator().translate(comment).text\n            # Create BERT features\n            comment_feature_ids, comment_features_masks = create_bert_input(tokenizer,  \n                                    comment, max_seq_len=MAX_SEQ_LENGTH)\n            # Employ the model to get the prediction and parse it\n            predicted_label = self.model.predict([comment_feature_ids, comment_features_masks])\n            predicted_label = np.argmax(predicted_label[0])\n            if predicted_label==0: predicted_label=\"Non-Toxic\"\n            else: predicted_label=\"Toxic\"\n            \n            sample = [comment, translated_comment, predicted_label]\n            \n            samples.append(sample)\n        wandb.log({\"text\": wandb.Table(data=samples, \n                                       columns=[\"Comment\", \"Translated Comment\", \"Predicted Label\"])})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Garbage collection\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Account for the class imbalance\nfrom sklearn.utils import class_weight\n\nclass_weights = class_weight.compute_class_weight('balanced',\n                                                 np.unique(y_train),\n                                                 y_train)\nclass_weights","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train the model\nimport time\n\nstart = time.time()\n\n# Compile the model with TPU Strategy\nwith strategy.scope():\n    model = get_training_model()\n    \nmodel.fit(train_ds, \n          steps_per_epoch=train_data.shape[0] // BATCH_SIZE,\n          validation_data=valid_ds,\n          validation_steps=val_data.shape[0] // BATCH_SIZE,\n          epochs=EPOCHS,\n          class_weight=class_weights,\n          callbacks=[WandbCallback(), TextLogger()],\n          verbose=1)\nend = time.time() - start\nprint(\"Time taken \",end)\nwandb.log({\"training_time\":end})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#sub['toxic'] = model.predict(test_ds, verbose=1)\nsub['toxic'] = model.predict([test_feature_ids, test_feature_masks], verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}