{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!wget --quiet https://raw.githubusercontent.com/tensorflow/models/master/official/nlp/bert/tokenization.py","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport os\nfrom tqdm.notebook import tqdm\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.callbacks import ModelCheckpoint\nimport tensorflow_hub as hub\nimport tokenization\nfrom sklearn.model_selection import train_test_split","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def roc_auc(predictions,target):\n    '''\n    This methods returns the AUC Score when given the Predictions\n    and Labels\n    '''\n    \n    fpr, tpr, thresholds = metrics.roc_curve(target, predictions)\n    roc_auc = metrics.auc(fpr, tpr)\n    return roc_auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Encode data for input to the model\ndef encode_text(texts, tokenizer, maxlen=512):\n    all_masks, all_tokens, all_segments = [], [], []\n    \n    for text in tqdm(texts):\n        text = tokenizer.tokenize(text)\n        text = text[:maxlen-2]\n        input_sq = [\"[CLS]\"] + text + [\"[SEP]\"]\n        pad_len = maxlen - len(input_sq)\n        tokens = tokenizer.convert_tokens_to_ids(input_sq)\n        tokens += [0] * pad_len\n        pad_masks = [1] * len(input_sq) + [0] * pad_len\n        segment_ids = [0] * maxlen\n        \n        all_masks.append(pad_masks)\n        all_tokens.append(tokens)\n        all_segments.append(segment_ids)\n    \n    return np.array(all_tokens), np.array(all_masks), np.array(all_segments)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ATO = tf.data.experimental.AUTOTUNE\nnb_epochs = 1\nbatch_size = 16 * strategy.num_replicas_in_sync\nmaxLen = 192","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nurl = \"https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/1\"\nbert_layer = hub.KerasLayer(url, trainable=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Get the vocab file (for tokenizing) and tokenizer itself\nvocab_fl = bert_layer.resolved_object.vocab_file.asset_path.numpy()\nlower_case = bert_layer.resolved_object.do_lower_case.numpy()\ntokenizer = tokenization.FullTokenizer(vocab_fl, lower_case)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input = encode_text(train['comment_text'].values, tokenizer, maxlen=192)\ntrain_labels = train['toxic'].values\nvalid_input = encode_text(valid['comment_text'].values, tokenizer, maxlen=192)\nvalid_labels = valid['toxic'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = encode_text(test['content'].values, tokenizer, maxlen=192)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    # Naming your keras ops is very important 😉\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name='input_word_ids')\n    input_mask = Input(shape=(max_len,), dtype=tf.int32, name='input_mask')\n    segment_ids = Input(shape=(max_len,), dtype=tf.int32, name='segment_ids')\n    # Get the sequence output\n    _, seq_op = transformer([input_word_ids, input_mask, segment_ids])\n    # Get the respective class token from that sequence output\n    class_tkn = seq_op[:, 0, :]\n    # Final Neuron (for Classification)\n    op = Dense(1, activation='sigmoid')(class_tkn)\n    # Bind the inputs and outputs together into a Model\n    model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=op)\n    \n    model.compile(optimizer=Adam(1e-5), loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(bert_layer, max_len=192)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = ModelCheckpoint('model.h5', monitor='val_loss', save_best_only=True)\n\ntrain_history = model.fit(\n    train_input, train_labels,\n    validation_data=(valid_input, valid_labels),\n    epochs=1,\n    callbacks=[checkpoint],\n    batch_size=16\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.save(\"final_model.h5\")","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}