{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n\n\nimport collections\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\nimport pandas as pd\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom datetime import datetime","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install bert-tensorflow","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import bert\nfrom bert import run_classifier\nfrom bert import optimization\nfrom bert import tokenization\nfrom bert import modeling","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#import tokenization\n#import modeling\nBERT_VOCAB= '../input/multi-cased/multi_cased_L-12_H-768_A-12/vocab.txt'\nBERT_INIT_CHKPNT = '../input/multi-cased/multi_cased_L-12_H-768_A-12/bert_model.ckpt'\nBERT_CONFIG = '../input/multi-cased/multi_cased_L-12_H-768_A-12/bert_config.json'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#tokenization.validate_case_matches_checkpoint(True,BERT_INIT_CHKPNT)\ntokenizer = tokenization.FullTokenizer(vocab_file=BERT_VOCAB,do_lower_case=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train_data_path='../input/jigsaw-toxic-comment-classification-challenge/train.csv'\ntrain = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntest = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"ID = 'id'\nDATA_COLUMN = 'comment_text'\nLABEL_COLUMNS = ['toxic','severe_toxic','obscene','threat','insult','identity_hate']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class InputExample(object):\n    \"\"\"A single training/test example for simple sequence classification.\"\"\"\n\n    def __init__(self, guid, text_a, text_b=None, labels=None):\n        \"\"\"Constructs a InputExample.\n\n        Args:\n            guid: Unique id for the example.\n            text_a: string. The untokenized text of the first sequence. For single\n            sequence tasks, only this sequence must be specified.\n            text_b: (Optional) string. The untokenized text of the second sequence.\n            Only must be specified for sequence pair tasks.\n            labels: (Optional) [string]. The label of the example. This should be\n            specified for train and dev examples, but not for test examples.\n        \"\"\"\n        self.guid = guid\n        self.text_a = text_a\n        self.text_b = text_b\n        self.labels = labels\n\n\nclass InputFeatures(object):\n    \"\"\"A single set of features of data.\"\"\"\n\n    def __init__(self, input_ids, input_mask, segment_ids, label_ids, is_real_example=True):\n        self.input_ids = input_ids\n        self.input_mask = input_mask\n        self.segment_ids = segment_ids\n        self.label_ids = label_ids,\n        self.is_real_example=is_real_example","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_examples(df, labels_available=True):\n    \"\"\"Creates examples for the training and dev sets.\"\"\"\n    examples = []\n    for (i, row) in enumerate(df.values):\n        guid = row[0]\n        text_a = row[1]\n        if labels_available:\n            labels = row[2:]\n        else:\n            labels = [0,0,0,0,0,0]\n        examples.append(\n            InputExample(guid=guid, text_a=text_a, labels=labels))\n    return examples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TRAIN_VAL_RATIO = 0.9\nLEN = train.shape[0]\nSIZE_TRAIN = int(TRAIN_VAL_RATIO*LEN)\n\nx_train = train[:SIZE_TRAIN]\nx_val = train[SIZE_TRAIN:]\n\n# Use the InputExample class from BERT's run_classifier code to create examples from the data\ntrain_examples = create_examples(x_train)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape, x_train.shape, x_val.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas\n\ndef convert_examples_to_features(examples,  max_seq_length, tokenizer):\n    \"\"\"Loads a data file into a list of `InputBatch`s.\"\"\"\n\n    features = []\n    for (ex_index, example) in enumerate(examples):\n        print(example.text_a)\n        tokens_a = tokenizer.tokenize(example.text_a)\n\n        tokens_b = None\n        if example.text_b:\n            tokens_b = tokenizer.tokenize(example.text_b)\n            # Modifies `tokens_a` and `tokens_b` in place so that the total\n            # length is less than the specified length.\n            # Account for [CLS], [SEP], [SEP] with \"- 3\"\n            _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)\n        else:\n            # Account for [CLS] and [SEP] with \"- 2\"\n            if len(tokens_a) > max_seq_length - 2:\n                tokens_a = tokens_a[:(max_seq_length - 2)]\n\n        # The convention in BERT is:\n        # (a) For sequence pairs:\n        #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]\n        #  type_ids: 0   0  0    0    0     0       0 0    1  1  1  1   1 1\n        # (b) For single sequences:\n        #  tokens:   [CLS] the dog is hairy . [SEP]\n        #  type_ids: 0   0   0   0  0     0 0\n        #\n        # Where \"type_ids\" are used to indicate whether this is the first\n        # sequence or the second sequence. The embedding vectors for `type=0` and\n        # `type=1` were learned during pre-training and are added to the wordpiece\n        # embedding vector (and position vector). This is not *strictly* necessary\n        # since the [SEP] token unambigiously separates the sequences, but it makes\n        # it easier for the model to learn the concept of sequences.\n        #\n        # For classification tasks, the first vector (corresponding to [CLS]) is\n        # used as as the \"sentence vector\". Note that this only makes sense because\n        # the entire model is fine-tuned.\n        tokens = [\"[CLS]\"] + tokens_a + [\"[SEP]\"]\n        segment_ids = [0] * len(tokens)\n\n        if tokens_b:\n            tokens += tokens_b + [\"[SEP]\"]\n            segment_ids += [1] * (len(tokens_b) + 1)\n\n        input_ids = tokenizer.convert_tokens_to_ids(tokens)\n\n        # The mask has 1 for real tokens and 0 for padding tokens. Only real\n        # tokens are attended to.\n        input_mask = [1] * len(input_ids)\n\n        # Zero-pad up to the sequence length.\n        padding = [0] * (max_seq_length - len(input_ids))\n        input_ids += padding\n        input_mask += padding\n        segment_ids += padding\n\n        assert len(input_ids) == max_seq_length\n        assert len(input_mask) == max_seq_length\n        assert len(segment_ids) == max_seq_length\n        \n        labels_ids = []\n        for label in example.labels:\n            labels_ids.append(int(label))\n\n        if ex_index < 0:\n            logger.info(\"*** Example ***\")\n            logger.info(\"guid: %s\" % (example.guid))\n            logger.info(\"tokens: %s\" % \" \".join(\n                    [str(x) for x in tokens]))\n            logger.info(\"input_ids: %s\" % \" \".join([str(x) for x in input_ids]))\n            logger.info(\"input_mask: %s\" % \" \".join([str(x) for x in input_mask]))\n            logger.info(\n                    \"segment_ids: %s\" % \" \".join([str(x) for x in segment_ids]))\n            logger.info(\"label: %s (id = %s)\" % (example.labels, labels_ids))\n\n        features.append(\n                InputFeatures(input_ids=input_ids,\n                              input_mask=input_mask,\n                              segment_ids=segment_ids,\n                              label_ids=labels_ids))\n    return features","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We'll set sequences to be at most 128 tokens long.\nMAX_SEQ_LENGTH = 128","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_model(bert_config, is_training, input_ids, input_mask, segment_ids,\n                 labels, num_labels, use_one_hot_embeddings):\n    \"\"\"Creates a classification model.\"\"\"\n    model = modeling.BertModel(\n        config=bert_config,\n        is_training=is_training,\n        input_ids=input_ids,\n        input_mask=input_mask,\n        token_type_ids=segment_ids,\n        use_one_hot_embeddings=use_one_hot_embeddings)\n\n    # In the demo, we are doing a simple classification task on the entire\n    # segment.\n    #\n    # If you want to use the token-level output, use model.get_sequence_output()\n    # instead.\n    output_layer = model.get_pooled_output()\n\n    hidden_size = output_layer.shape[-1].value\n\n    output_weights = tf.get_variable(\n        \"output_weights\", [num_labels, hidden_size],\n        initializer=tf.truncated_normal_initializer(stddev=0.02))\n\n    output_bias = tf.get_variable(\n        \"output_bias\", [num_labels], initializer=tf.zeros_initializer())\n\n    with tf.variable_scope(\"loss\"):\n        if is_training:\n            # I.e., 0.1 dropout\n            output_layer = tf.nn.dropout(output_layer, keep_prob=0.9)\n\n        logits = tf.matmul(output_layer, output_weights, transpose_b=True)\n        logits = tf.nn.bias_add(logits, output_bias)\n        \n        # probabilities = tf.nn.softmax(logits, axis=-1) ### multiclass case\n        probabilities = tf.nn.sigmoid(logits)#### multi-label case\n        \n        labels = tf.cast(labels, tf.float32)\n        tf.logging.info(\"num_labels:{};logits:{};labels:{}\".format(num_labels, logits, labels))\n        per_example_loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels, logits=logits)\n        loss = tf.reduce_mean(per_example_loss)\n\n        # probabilities = tf.nn.softmax(logits, axis=-1)\n        # log_probs = tf.nn.log_softmax(logits, axis=-1)\n        #\n        # one_hot_labels = tf.one_hot(labels, depth=num_labels, dtype=tf.float32)\n        #\n        # per_example_loss = -tf.reduce_sum(one_hot_labels * log_probs, axis=-1)\n        # loss = tf.reduce_mean(per_example_loss)\n\n        return (loss, per_example_loss, logits, probabilities)\n\n\ndef model_fn_builder(bert_config, num_labels, init_checkpoint, learning_rate,\n                     num_train_steps, num_warmup_steps, use_tpu,\n                     use_one_hot_embeddings):\n    \"\"\"Returns `model_fn` closure for TPUEstimator.\"\"\"\n\n    def model_fn(features, labels, mode, params):  # pylint: disable=unused-argument\n        \"\"\"The `model_fn` for TPUEstimator.\"\"\"\n\n        #tf.logging.info(\"*** Features ***\")\n        #for name in sorted(features.keys()):\n        #    tf.logging.info(\"  name = %s, shape = %s\" % (name, features[name].shape))\n\n        input_ids = features[\"input_ids\"]\n        input_mask = features[\"input_mask\"]\n        segment_ids = features[\"segment_ids\"]\n        label_ids = features[\"label_ids\"]\n        is_real_example = None\n        if \"is_real_example\" in features:\n             is_real_example = tf.cast(features[\"is_real_example\"], dtype=tf.float32)\n        else:\n             is_real_example = tf.ones(tf.shape(label_ids), dtype=tf.float32)\n\n        is_training = (mode == tf.estimator.ModeKeys.TRAIN)\n\n        (total_loss, per_example_loss, logits, probabilities) = create_model(\n            bert_config, is_training, input_ids, input_mask, segment_ids, label_ids,\n            num_labels, use_one_hot_embeddings)\n\n        tvars = tf.trainable_variables()\n        initialized_variable_names = {}\n        scaffold_fn = None\n        if init_checkpoint:\n            (assignment_map, initialized_variable_names\n             ) = modeling.get_assignment_map_from_checkpoint(tvars, init_checkpoint)\n            if use_tpu:\n\n                def tpu_scaffold():\n                    tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n                    return tf.train.Scaffold()\n\n                scaffold_fn = tpu_scaffold\n            else:\n                tf.train.init_from_checkpoint(init_checkpoint, assignment_map)\n\n        tf.logging.info(\"**** Trainable Variables ****\")\n        for var in tvars:\n            init_string = \"\"\n            if var.name in initialized_variable_names:\n                init_string = \", *INIT_FROM_CKPT*\"\n            #tf.logging.info(\"  name = %s, shape = %s%s\", var.name, var.shape,init_string)\n\n        output_spec = None\n        if mode == tf.estimator.ModeKeys.TRAIN:\n\n            train_op = optimization.create_optimizer(\n                total_loss, learning_rate, num_train_steps, num_warmup_steps, use_tpu)\n\n            output_spec = tf.estimator.EstimatorSpec(\n                mode=mode,\n                loss=total_loss,\n                train_op=train_op,\n                scaffold=scaffold_fn)\n        elif mode == tf.estimator.ModeKeys.EVAL:\n\n            def metric_fn(per_example_loss, label_ids, probabilities, is_real_example):\n\n                logits_split = tf.split(probabilities, num_labels, axis=-1)\n                label_ids_split = tf.split(label_ids, num_labels, axis=-1)\n                # metrics change to auc of every class\n                eval_dict = {}\n                for j, logits in enumerate(logits_split):\n                    label_id_ = tf.cast(label_ids_split[j], dtype=tf.int32)\n                    current_auc, update_op_auc = tf.metrics.auc(label_id_, logits)\n                    eval_dict[str(j)] = (current_auc, update_op_auc)\n                eval_dict['eval_loss'] = tf.metrics.mean(values=per_example_loss)\n                return eval_dict\n\n                ## original eval metrics\n                # predictions = tf.argmax(logits, axis=-1, output_type=tf.int32)\n                # accuracy = tf.metrics.accuracy(\n                #     labels=label_ids, predictions=predictions, weights=is_real_example)\n                # loss = tf.metrics.mean(values=per_example_loss, weights=is_real_example)\n                # return {\n                #     \"eval_accuracy\": accuracy,\n                #     \"eval_loss\": loss,\n                # }\n\n            eval_metrics = metric_fn(per_example_loss, label_ids, probabilities, is_real_example)\n            output_spec = tf.estimator.EstimatorSpec(\n                mode=mode,\n                loss=total_loss,\n                eval_metric_ops=eval_metrics,\n                scaffold=scaffold_fn)\n        else:\n            print(\"mode:\", mode,\"probabilities:\", probabilities)\n            output_spec = tf.estimator.EstimatorSpec(\n                mode=mode,\n                predictions={\"probabilities\": probabilities},\n                scaffold=scaffold_fn)\n        return output_spec\n\n    return model_fn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compute train and warmup steps from batch size\n# These hyperparameters are copied from this colab notebook (https://colab.sandbox.google.com/github/tensorflow/tpu/blob/master/tools/colab/bert_finetuning_with_cloud_tpus.ipynb)\nBATCH_SIZE = 32\nLEARNING_RATE = 2e-5\nNUM_TRAIN_EPOCHS = 2.0\n\n# Warmup is a period of time where hte learning rate \n# is small and gradually increases--usually helps training.\nWARMUP_PROPORTION = 0.1\n# Model configs\nSAVE_CHECKPOINTS_STEPS = 1000\nSAVE_SUMMARY_STEPS = 500","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"OUTPUT_DIR = \"../working/output\"\n# Specify outpit directory and number of checkpoint steps to save\nrun_config = tf.estimator.RunConfig(\n    model_dir=OUTPUT_DIR,\n    save_summary_steps=SAVE_SUMMARY_STEPS,\n    keep_checkpoint_max=1,\n    save_checkpoints_steps=SAVE_CHECKPOINTS_STEPS)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def input_fn_builder(features, seq_length, is_training, drop_remainder):\n  \"\"\"Creates an `input_fn` closure to be passed to TPUEstimator.\"\"\"\n\n  all_input_ids = []\n  all_input_mask = []\n  all_segment_ids = []\n  all_label_ids = []\n\n  for feature in features:\n    all_input_ids.append(feature.input_ids)\n    all_input_mask.append(feature.input_mask)\n    all_segment_ids.append(feature.segment_ids)\n    all_label_ids.append(feature.label_ids)\n\n  def input_fn(params):\n    \"\"\"The actual input function.\"\"\"\n    batch_size = params[\"batch_size\"]\n\n    num_examples = len(features)\n\n    # This is for demo purposes and does NOT scale to large data sets. We do\n    # not use Dataset.from_generator() because that uses tf.py_func which is\n    # not TPU compatible. The right way to load data is with TFRecordReader.\n    d = tf.data.Dataset.from_tensor_slices({\n        \"input_ids\":\n            tf.constant(\n                all_input_ids, shape=[num_examples, seq_length],\n                dtype=tf.int32),\n        \"input_mask\":\n            tf.constant(\n                all_input_mask,\n                shape=[num_examples, seq_length],\n                dtype=tf.int32),\n        \"segment_ids\":\n            tf.constant(\n                all_segment_ids,\n                shape=[num_examples, seq_length],\n                dtype=tf.int32),\n        \"label_ids\":\n            tf.constant(all_label_ids, shape=[num_examples, len(LABEL_COLUMNS)], dtype=tf.int32),\n    })\n\n    if is_training:\n      d = d.repeat()\n      d = d.shuffle(buffer_size=100)\n\n    d = d.batch(batch_size=batch_size, drop_remainder=drop_remainder)\n    return d\n\n  return input_fn\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class PaddingInputExample(object):\n    \"\"\"Fake example so the num input examples is a multiple of the batch size.\n    When running eval/predict on the TPU, we need to pad the number of examples\n    to be a multiple of the batch size, because the TPU requires a fixed batch\n    size. The alternative is to drop the last batch, which is bad because it means\n    the entire output data won't be generated.\n    We use this class instead of `None` because treating `None` as padding\n    battches could cause silent errors.\n    \"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def convert_single_example(ex_index, example, max_seq_length,\n                           tokenizer):\n    \"\"\"Converts a single `InputExample` into a single `InputFeatures`.\"\"\"\n\n    if isinstance(example, PaddingInputExample):\n        return InputFeatures(\n            input_ids=[0] * max_seq_length,\n            input_mask=[0] * max_seq_length,\n            segment_ids=[0] * max_seq_length,\n            label_ids=0,\n            is_real_example=False)\n\n    tokens_a = tokenizer.tokenize(example.text_a)\n    tokens_b = None\n    if example.text_b:\n        tokens_b = tokenizer.tokenize(example.text_b)\n\n    if tokens_b:\n        # Modifies `tokens_a` and `tokens_b` in place so that the total\n        # length is less than the specified length.\n        # Account for [CLS], [SEP], [SEP] with \"- 3\"\n        _truncate_seq_pair(tokens_a, tokens_b, max_seq_length - 3)\n    else:\n        # Account for [CLS] and [SEP] with \"- 2\"\n        if len(tokens_a) > max_seq_length - 2:\n            tokens_a = tokens_a[0:(max_seq_length - 2)]\n\n    # The convention in BERT is:\n    # (a) For sequence pairs:\n    #  tokens:   [CLS] is this jack ##son ##ville ? [SEP] no it is not . [SEP]\n    #  type_ids: 0     0  0    0    0     0       0 0     1  1  1  1   1 1\n    # (b) For single sequences:\n    #  tokens:   [CLS] the dog is hairy . [SEP]\n    #  type_ids: 0     0   0   0  0     0 0\n    #\n    # Where \"type_ids\" are used to indicate whether this is the first\n    # sequence or the second sequence. The embedding vectors for `type=0` and\n    # `type=1` were learned during pre-training and are added to the wordpiece\n    # embedding vector (and position vector). This is not *strictly* necessary\n    # since the [SEP] token unambiguously separates the sequences, but it makes\n    # it easier for the model to learn the concept of sequences.\n    #\n    # For classification tasks, the first vector (corresponding to [CLS]) is\n    # used as the \"sentence vector\". Note that this only makes sense because\n    # the entire model is fine-tuned.\n    tokens = []\n    segment_ids = []\n    tokens.append(\"[CLS]\")\n    segment_ids.append(0)\n    for token in tokens_a:\n        tokens.append(token)\n        segment_ids.append(0)\n    tokens.append(\"[SEP]\")\n    segment_ids.append(0)\n\n    if tokens_b:\n        for token in tokens_b:\n            tokens.append(token)\n            segment_ids.append(1)\n        tokens.append(\"[SEP]\")\n        segment_ids.append(1)\n\n    input_ids = tokenizer.convert_tokens_to_ids(tokens)\n\n    # The mask has 1 for real tokens and 0 for padding tokens. Only real\n    # tokens are attended to.\n    input_mask = [1] * len(input_ids)\n\n    # Zero-pad up to the sequence length.\n    while len(input_ids) < max_seq_length:\n        input_ids.append(0)\n        input_mask.append(0)\n        segment_ids.append(0)\n\n    assert len(input_ids) == max_seq_length\n    assert len(input_mask) == max_seq_length\n    assert len(segment_ids) == max_seq_length\n\n    labels_ids = []\n    for label in example.labels:\n        labels_ids.append(int(label))\n\n\n    feature = InputFeatures(\n        input_ids=input_ids,\n        input_mask=input_mask,\n        segment_ids=segment_ids,\n        label_ids=labels_ids,\n        is_real_example=True)\n    return feature\n\n\ndef file_based_convert_examples_to_features(\n        examples, max_seq_length, tokenizer, output_file):\n    \"\"\"Convert a set of `InputExample`s to a TFRecord file.\"\"\"\n\n    writer = tf.python_io.TFRecordWriter(output_file)\n\n    for (ex_index, example) in enumerate(examples):\n        #if ex_index % 10000 == 0:\n            #tf.logging.info(\"Writing example %d of %d\" % (ex_index, len(examples)))\n\n        feature = convert_single_example(ex_index, example,\n                                         max_seq_length, tokenizer)\n\n        def create_int_feature(values):\n            f = tf.train.Feature(int64_list=tf.train.Int64List(value=list(values)))\n            return f\n\n        features = collections.OrderedDict()\n        features[\"input_ids\"] = create_int_feature(feature.input_ids)\n        features[\"input_mask\"] = create_int_feature(feature.input_mask)\n        features[\"segment_ids\"] = create_int_feature(feature.segment_ids)\n        features[\"is_real_example\"] = create_int_feature(\n            [int(feature.is_real_example)])\n        if isinstance(feature.label_ids, list):\n            label_ids = feature.label_ids\n        else:\n            label_ids = feature.label_ids[0]\n        features[\"label_ids\"] = create_int_feature(label_ids)\n\n        tf_example = tf.train.Example(features=tf.train.Features(feature=features))\n        writer.write(tf_example.SerializeToString())\n    writer.close()\n\n\ndef file_based_input_fn_builder(input_file, seq_length, is_training,\n                                drop_remainder):\n    \"\"\"Creates an `input_fn` closure to be passed to TPUEstimator.\"\"\"\n\n    name_to_features = {\n        \"input_ids\": tf.FixedLenFeature([seq_length], tf.int64),\n        \"input_mask\": tf.FixedLenFeature([seq_length], tf.int64),\n        \"segment_ids\": tf.FixedLenFeature([seq_length], tf.int64),\n        \"label_ids\": tf.FixedLenFeature([6], tf.int64),\n        \"is_real_example\": tf.FixedLenFeature([], tf.int64),\n    }\n\n    def _decode_record(record, name_to_features):\n        \"\"\"Decodes a record to a TensorFlow example.\"\"\"\n        example = tf.parse_single_example(record, name_to_features)\n\n        # tf.Example only supports tf.int64, but the TPU only supports tf.int32.\n        # So cast all int64 to int32.\n        for name in list(example.keys()):\n            t = example[name]\n            if t.dtype == tf.int64:\n                t = tf.to_int32(t)\n            example[name] = t\n\n        return example\n\n    def input_fn(params):\n        \"\"\"The actual input function.\"\"\"\n        batch_size = params[\"batch_size\"]\n\n        # For training, we want a lot of parallel reading and shuffling.\n        # For eval, we want no shuffling and parallel reading doesn't matter.\n        d = tf.data.TFRecordDataset(input_file)\n        if is_training:\n            d = d.repeat()\n            d = d.shuffle(buffer_size=100)\n\n        d = d.apply(\n            tf.contrib.data.map_and_batch(\n                lambda record: _decode_record(record, name_to_features),\n                batch_size=batch_size,\n                drop_remainder=drop_remainder))\n\n        return d\n\n    return input_fn\n\n\ndef _truncate_seq_pair(tokens_a, tokens_b, max_length):\n    \"\"\"Truncates a sequence pair in place to the maximum length.\"\"\"\n\n    # This is a simple heuristic which will always truncate the longer sequence\n    # one token at a time. This makes more sense than truncating an equal percent\n    # of tokens from each, since if one sequence is very short then each token\n    # that's truncated likely contains more information than a longer sequence.\n    while True:\n        total_length = len(tokens_a) + len(tokens_b)\n        if total_length <= max_length:\n            break\n        if len(tokens_a) > len(tokens_b):\n            tokens_a.pop()\n        else:\n            tokens_b.pop()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from pathlib import Path\ntrain_file = os.path.join('../working', \"train.tf_record\")\n#filename = Path(train_file)\nif not os.path.exists(train_file):\n    open(train_file, 'w').close()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"train_features = convert_examples_to_features(\n    train_examples, MAX_SEQ_LENGTH, tokenizer)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# Create an input function for training. drop_remainder = True for using TPUs.\ntrain_input_fn = input_fn_builder(\n    features=train_features,\n    seq_length=MAX_SEQ_LENGTH,\n    is_training=True,\n    drop_remainder=False)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Compute # train and warmup steps from batch size\nnum_train_steps = int(len(train_examples) / BATCH_SIZE * NUM_TRAIN_EPOCHS)\nnum_warmup_steps = int(num_train_steps * WARMUP_PROPORTION)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"file_based_convert_examples_to_features(\n            train_examples, MAX_SEQ_LENGTH, tokenizer, train_file)\ntf.logging.info(\"***** Running training *****\")\ntf.logging.info(\"  Num examples = %d\", len(train_examples))\ntf.logging.info(\"  Batch size = %d\", BATCH_SIZE)\ntf.logging.info(\"  Num steps = %d\", num_train_steps)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_input_fn = file_based_input_fn_builder(\n    input_file=train_file,\n    seq_length=MAX_SEQ_LENGTH,\n    is_training=True,\n    drop_remainder=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bert_config = modeling.BertConfig.from_json_file(BERT_CONFIG)\nmodel_fn = model_fn_builder(\n  bert_config=bert_config,\n  num_labels= len(LABEL_COLUMNS),\n  init_checkpoint=BERT_INIT_CHKPNT,\n  learning_rate=LEARNING_RATE,\n  num_train_steps=num_train_steps,\n  num_warmup_steps=num_warmup_steps,\n  use_tpu=False,\n  use_one_hot_embeddings=False)\n\nestimator = tf.estimator.Estimator(\n  model_fn=model_fn,\n  config=run_config,\n  params={\"batch_size\": BATCH_SIZE})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f'Beginning Training!')\ncurrent_time = datetime.now()\nestimator.train(input_fn=train_input_fn, max_steps=num_train_steps)\nprint(\"Training took time \", datetime.now() - current_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eval_file = os.path.join('../working', \"eval.tf_record\")\n#filename = Path(train_file)\nif not os.path.exists(eval_file):\n    open(eval_file, 'w').close()\n\neval_examples = create_examples(x_val)\nfile_based_convert_examples_to_features(\n    eval_examples, MAX_SEQ_LENGTH, tokenizer, eval_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# This tells the estimator to run through the entire set.\neval_steps = None\n\neval_drop_remainder = False\neval_input_fn = file_based_input_fn_builder(\n    input_file=eval_file,\n    seq_length=MAX_SEQ_LENGTH,\n    is_training=False,\n    drop_remainder=False)\n\nresult = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":false},"cell_type":"markdown","source":"#x_eval = train[100000:]\n# Use the InputExample class from BERT's run_classifier code to create examples from the data\neval_examples = create_examples(x_val)\n\neval_features = convert_examples_to_features(\n    eval_examples, MAX_SEQ_LENGTH, tokenizer)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"# This tells the estimator to run through the entire set.\neval_steps = None\n\neval_drop_remainder = False\neval_input_fn = input_fn_builder(\n    features=eval_features,\n    seq_length=MAX_SEQ_LENGTH,\n    is_training=False,\n    drop_remainder=eval_drop_remainder)\n\nresult = estimator.evaluate(input_fn=eval_input_fn, steps=eval_steps)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"output_eval_file = os.path.join(\"../working\", \"eval_results.txt\")\nwith tf.gfile.GFile(output_eval_file, \"w\") as writer:\n    tf.logging.info(\"***** Eval results *****\")\n    for key in sorted(result.keys()):\n        tf.logging.info(\"  %s = %s\", key, str(result[key]))\n        writer.write(\"%s = %s\\n\" % (key, str(result[key])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = test#[125000:140000]\nx_test = x_test.reset_index(drop=True)\n\ntest_file = os.path.join('../working', \"test.tf_record\")\n#filename = Path(train_file)\nif not os.path.exists(test_file):\n    open(test_file, 'w').close()\n\ntest_examples = create_examples(x_test, False)\nfile_based_convert_examples_to_features(\n    test_examples, MAX_SEQ_LENGTH, tokenizer, test_file)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"predict_input_fn = file_based_input_fn_builder(\n    input_file=test_file,\n    seq_length=MAX_SEQ_LENGTH,\n    is_training=False,\n    drop_remainder=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Begin predictions!')\ncurrent_time = datetime.now()\npredictions = estimator.predict(predict_input_fn)\nprint(\"Predicting took time \", datetime.now() - current_time)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"x_test = test[125000:140000]\nx_test = x_test.reset_index(drop=True)\npredict_examples = create_examples(x_test,False)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"test_features = convert_examples_to_features(predict_examples, MAX_SEQ_LENGTH, tokenizer)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"print(f'Beginning Training!')\ncurrent_time = datetime.now()\n\npredict_input_fn = input_fn_builder(features=test_features, seq_length=MAX_SEQ_LENGTH, is_training=False, drop_remainder=False)\npredictions = estimator.predict(predict_input_fn)\nprint(\"Training took time \", datetime.now() - current_time)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def create_output(predictions):\n    probabilities = []\n    for (i, prediction) in enumerate(predictions):\n        preds = prediction[\"probabilities\"]\n        probabilities.append(preds)\n    dff = pd.DataFrame(probabilities)\n    dff.columns = LABEL_COLUMNS\n    \n    return dff\n        ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"output_df = create_output(predictions)\nmerged_df =  pd.concat([x_test, output_df], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"merged_df.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission = merged_df.drop(['content','lang','severe_toxic','obscene','threat','insult','identity_hate'], axis=1)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.to_csv(\"sample_submission.csv\", index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}