{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import codecs\nimport copy\nimport csv\nimport gc\nimport os\nimport pickle\nimport random\nimport time\nfrom typing import Dict, List, Sequence, Set, Tuple","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn.manifold import TSNE\nfrom sklearn.metrics import roc_auc_score, roc_curve\nimport tensorflow as tf\nfrom tensorflow.python.framework import ops, tensor_util\nfrom tensorflow.python.keras.utils import losses_utils, tf_utils\nfrom tensorflow.python.ops import math_ops\nfrom tensorflow.python.ops.losses import util as tf_losses_util\nimport tensorflow_addons as tfa\nfrom transformers import AutoTokenizer, XLMRobertaTokenizer\nfrom transformers import TFXLMRobertaModel, XLMRobertaConfig","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class LossFunctionWrapper(tf.keras.losses.Loss):\n    def __init__(self,\n                 fn,\n                 reduction=losses_utils.ReductionV2.AUTO,\n                 name=None,\n                 **kwargs):\n        super(LossFunctionWrapper, self).__init__(reduction=reduction, name=name)\n        self.fn = fn\n        self._fn_kwargs = kwargs\n\n    def call(self, y_true, y_pred):\n        if tensor_util.is_tensor(y_pred) and tensor_util.is_tensor(y_true):\n            y_pred, y_true = tf_losses_util.squeeze_or_expand_dimensions(y_pred, y_true)\n        return self.fn(y_true, y_pred, **self._fn_kwargs)\n\n    def get_config(self):\n        config = {}\n        for k, v in six.iteritems(self._fn_kwargs):\n            config[k] = tf.keras.backend.eval(v) if tf_utils.is_tensor_or_variable(v) \\\n                else v\n        base_config = super(LossFunctionWrapper, self).get_config()\n        return dict(list(base_config.items()) + list(config.items()))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def distance_based_log_loss(y_true, y_pred):\n    y_pred = ops.convert_to_tensor(y_pred)\n    y_true = math_ops.cast(y_true, y_pred.dtype)\n    margin = 1.0\n    p = (1.0 + tf.math.exp(-margin)) / (1.0 + tf.math.exp(y_pred - margin))\n    return tf.keras.backend.binary_crossentropy(target=y_true, output=p)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DBLLogLoss(LossFunctionWrapper):\n    def __init__(self, reduction=losses_utils.ReductionV2.AUTO,\n                 name='distance_based_log_loss'):\n        super(DBLLogLoss, self).__init__(distance_based_log_loss, name=name,\n                                         reduction=reduction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dice_loss(y_true, y_pred):\n    y_pred = ops.convert_to_tensor(y_pred)\n    y_true = math_ops.cast(y_true, y_pred.dtype)\n    dice_loss_1 = tf.keras.backend.mean(y_true * y_pred, axis=-1)\n    dice_loss_2 = tf.keras.backend.mean(y_true * y_true, axis=-1)\n    dice_loss_3 = tf.keras.backend.mean(y_pred * y_pred, axis=-1)\n    return 1.0 - (2 * dice_loss_1 + 1.0) / (dice_loss_2 + dice_loss_3 + 1.0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class DiceLoss(LossFunctionWrapper):\n    def __init__(self, reduction=losses_utils.ReductionV2.AUTO, name='dice_loss'):\n        super(DiceLoss, self).__init__(dice_loss, name=name, reduction=reduction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class MaskCalculator(tf.keras.layers.Layer):\n    def __init__(self, output_dim, **kwargs):\n        self.output_dim = output_dim\n        super(MaskCalculator, self).__init__(**kwargs)\n\n    def build(self, input_shape):\n        super(MaskCalculator, self).build(input_shape)\n\n    def call(self, inputs, **kwargs):\n        return tf.keras.backend.permute_dimensions(\n            x=tf.keras.backend.repeat(\n                x=tf.keras.backend.cast(\n                    x=tf.keras.backend.greater(\n                        x=inputs,\n                        y=0\n                    ),\n                    dtype='float32'\n                ),\n                n=self.output_dim\n            ),\n            pattern=(0, 2, 1)\n        )\n\n    def compute_output_shape(self, input_shape):\n        assert len(input_shape) == 1\n        shape = list(input_shape)\n        shape.append(self.output_dim)\n        return tuple(shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts: List[str], tokenizer: XLMRobertaTokenizer,\n                   maxlen: int) -> Tuple[np.ndarray, np.ndarray]:\n    enc_di = tokenizer.batch_encode_plus(\n        texts,\n        return_attention_masks=True,\n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    return np.array(enc_di['input_ids']), np.array(enc_di['attention_mask'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_train_set(file_name: str, text_field: str, sentiment_fields: List[str],\n                   lang_field: str) -> Dict[str, List[Tuple[str, int]]]:\n    assert len(sentiment_fields) > 0, 'List of sentiment fields is empty!'\n    header = []\n    line_idx = 1\n    data_by_lang = dict()\n    with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore') as fp:\n        data_reader = csv.reader(fp, quotechar='\"', delimiter=',')\n        for row in data_reader:\n            if len(row) > 0:\n                err_msg = 'File \"{0}\": line {1} is wrong!'.format(file_name, line_idx)\n                if len(header) == 0:\n                    header = copy.copy(row)\n                    err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(text_field)\n                    assert text_field in header, err_msg2\n                    for cur_field in sentiment_fields:\n                        err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(\n                            cur_field)\n                        assert cur_field in header, err_msg2\n                    text_field_index = header.index(text_field)\n                    try:\n                        lang_field_index = header.index(lang_field)\n                    except:\n                        lang_field_index = -1\n                    indices_of_sentiment_fields = []\n                    for cur_field in sentiment_fields:\n                        indices_of_sentiment_fields.append(header.index(cur_field))\n                else:\n                    if len(row) == len(header):\n                        text = row[text_field_index].strip()\n                        assert len(text) > 0, err_msg + ' Text is empty!'\n                        if lang_field_index >= 0:\n                            cur_lang = row[lang_field_index].strip()\n                            assert len(cur_lang) > 0, err_msg + ' Language is empty!'\n                        else:\n                            cur_lang = 'en'\n                        max_proba = 0.0\n                        for cur_field_idx in indices_of_sentiment_fields:\n                            try:\n                                cur_proba = float(row[cur_field_idx])\n                            except:\n                                cur_proba = -1.0\n                            err_msg2 = err_msg + ' Value {0} is wrong!'.format(\n                                row[cur_field_idx]\n                            )\n                            assert (cur_proba >= 0.0) and (cur_proba <= 1.0), err_msg2\n                            if cur_proba > max_proba:\n                                max_proba = cur_proba\n                        new_label = 1 if max_proba >= 0.5 else 0\n                        if cur_lang not in data_by_lang:\n                            data_by_lang[cur_lang] = []\n                        data_by_lang[cur_lang].append((text, new_label))\n            if line_idx % 10000 == 0:\n                print('{0} lines of the \"{1}\" have been processed...'.format(line_idx,\n                                                                             file_name))\n            line_idx += 1\n    if line_idx > 0:\n        if (line_idx - 1) % 10000 != 0:\n            print('{0} lines of the \"{1}\" have been processed...'.format(line_idx - 1,\n                                                                         file_name))\n    return data_by_lang","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def load_test_set(file_name: str, id_field: str, text_field: str,\n                  lang_field: str) -> Dict[str, List[Tuple[str, int]]]:\n    header = []\n    line_idx = 1\n    data_by_lang = dict()\n    with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore') as fp:\n        data_reader = csv.reader(fp, quotechar='\"', delimiter=',')\n        for row in data_reader:\n            if len(row) > 0:\n                err_msg = 'File \"{0}\": line {1} is wrong!'.format(file_name, line_idx)\n                if len(header) == 0:\n                    header = copy.copy(row)\n                    err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(text_field)\n                    assert text_field in header, err_msg2\n                    err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(id_field)\n                    assert id_field in header, err_msg2\n                    err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(lang_field)\n                    assert lang_field in header, err_msg2\n                    id_field_index = header.index(id_field)\n                    text_field_index = header.index(text_field)\n                    lang_field_index = header.index(lang_field)\n                else:\n                    if len(row) == len(header):\n                        try:\n                            id_value = int(row[id_field_index])\n                        except:\n                            id_value = -1\n                        err_msg2 = err_msg + ' {0} is wrong ID!'.format(\n                            row[id_field_index])\n                        assert id_value >= 0, err_msg2\n                        text = row[text_field_index].strip()\n                        assert len(text) > 0, err_msg + ' Text is empty!'\n                        if lang_field_index >= 0:\n                            cur_lang = row[lang_field_index].strip()\n                            assert len(cur_lang) > 0, err_msg + ' Language is empty!'\n                        else:\n                            cur_lang = 'en'\n                        if cur_lang not in data_by_lang:\n                            data_by_lang[cur_lang] = []\n                        data_by_lang[cur_lang].append((text, id_value))\n            if line_idx % 10000 == 0:\n                print('{0} lines of the \"{1}\" have been processed...'.format(line_idx,\n                                                                             file_name))\n            line_idx += 1\n    if line_idx > 0:\n        if (line_idx - 1) % 10000 != 0:\n            print('{0} lines of the \"{1}\" have been processed...'.format(line_idx - 1,\n                                                                         file_name))\n    return data_by_lang","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_siamese_dataset(texts: Dict[str, List[Tuple[str, int]]],\n                          dataset_size: int, tokenizer: XLMRobertaTokenizer,\n                          maxlen: int, batch_size: int,\n                          for_training: bool) -> Tuple[tf.data.Dataset, int]:\n    language_pairs = set()\n    for language in texts.keys():\n        for other_language in texts:\n            if other_language == language:\n                language_pairs.add((language, other_language))\n            else:\n                new_pair = (language, other_language)\n                new_pair_2 = (other_language, language)\n                if (new_pair not in language_pairs) and (new_pair_2 not in language_pairs):\n                    language_pairs.add(new_pair)\n    language_pairs = sorted(list(language_pairs))\n    print('Possible language pairs are: {0}.'.format(language_pairs))\n    err_msg = '{0} is too small size of the data set!'.format(dataset_size)\n    assert dataset_size >= (len(language_pairs) * 10), err_msg\n    n_samples_for_lang_pair = int(np.ceil(dataset_size / float(len(language_pairs))))\n    text_pairs_and_labels = []\n    for left_lang, right_lang in language_pairs:\n        print('{0}-{1}:'.format(left_lang, right_lang))\n        left_positive_indices = list(filter(\n            lambda idx: texts[left_lang][idx][1] > 0, range(len(texts[left_lang]))\n        ))\n        left_negative_indices = list(filter(\n            lambda idx: texts[left_lang][idx][1] == 0, range(len(texts[left_lang]))\n        ))\n        right_positive_indices = list(filter(\n            lambda idx: texts[right_lang][idx][1] > 0, range(len(texts[right_lang]))\n        ))\n        right_negative_indices = list(filter(\n            lambda idx: texts[right_lang][idx][1] == 0, range(len(texts[right_lang]))\n        ))\n        used_pairs = set()\n        number_of_samples = 0\n        for _ in range(n_samples_for_lang_pair // 4):\n            left_idx = random.choice(left_positive_indices)\n            right_idx = random.choice(right_positive_indices)\n            counter = 0\n            while ((right_idx == left_idx) or ((left_idx, right_idx) in used_pairs) or\n                   ((right_idx, left_idx) in used_pairs)) and (counter < 100):\n                right_idx = random.choice(right_positive_indices)\n                counter += 1\n            if counter < 100:\n                used_pairs.add((left_idx, right_idx))\n                used_pairs.add((right_idx, left_idx))\n                text_pairs_and_labels.append(\n                    (\n                        texts[left_lang][left_idx][0],\n                        texts[right_lang][right_idx][0],\n                        1,\n                        1,\n                        1\n                    )\n                )\n                number_of_samples += 1\n        print('  number of \"1-1\" pairs is {0};'.format(number_of_samples))\n        number_of_samples = 0\n        for _ in range(n_samples_for_lang_pair // 4, (2 * n_samples_for_lang_pair) // 4):\n            left_idx = random.choice(left_negative_indices)\n            right_idx = random.choice(right_negative_indices)\n            counter = 0\n            while ((right_idx == left_idx) or ((left_idx, right_idx) in used_pairs) or\n                   ((right_idx, left_idx) in used_pairs)) and (counter < 100):\n                right_idx = random.choice(right_negative_indices)\n                counter += 1\n            if counter < 100:\n                used_pairs.add((left_idx, right_idx))\n                used_pairs.add((right_idx, left_idx))\n                text_pairs_and_labels.append(\n                    (\n                        texts[left_lang][left_idx][0],\n                        texts[right_lang][right_idx][0],\n                        1,\n                        0,\n                        0\n                    )\n                )\n                number_of_samples += 1\n        print('  number of \"0-0\" pairs is {0};'.format(number_of_samples))\n        number_of_samples = 0\n        for _ in range((2 * n_samples_for_lang_pair) // 4, n_samples_for_lang_pair):\n            left_idx = random.choice(left_negative_indices)\n            right_idx = random.choice(right_positive_indices)\n            counter = 0\n            while ((right_idx == left_idx) or ((left_idx, right_idx) in used_pairs) or\n                   ((right_idx, left_idx) in used_pairs)) and (counter < 100):\n                right_idx = random.choice(right_positive_indices)\n                counter += 1\n            if counter < 100:\n                used_pairs.add((left_idx, right_idx))\n                used_pairs.add((right_idx, left_idx))\n                if random.random() >= 0.5:\n                    text_pairs_and_labels.append(\n                        (\n                            texts[left_lang][left_idx][0],\n                            texts[right_lang][right_idx][0],\n                            0,\n                            0,\n                            1\n                        )\n                    )\n                else:\n                    text_pairs_and_labels.append(\n                        (\n                            texts[right_lang][right_idx][0],\n                            texts[left_lang][left_idx][0],\n                            0,\n                            1,\n                            0\n                        )\n                    )\n                number_of_samples += 1\n        print('  number of \"0-1\" or \"1-0\" pairs is {0}.'.format(number_of_samples))\n    random.shuffle(text_pairs_and_labels)\n    n_steps = len(text_pairs_and_labels) // batch_size\n    print('Samples number of the data set is {0}.'.format(len(text_pairs_and_labels)))\n    print('Samples number per each language pair is {0}.'.format(n_samples_for_lang_pair))\n    tokens_of_left_texts, mask_of_left_texts = regular_encode(\n        texts=[cur[0] for cur in text_pairs_and_labels],\n        tokenizer=tokenizer, maxlen=maxlen\n    )\n    tokens_of_right_texts, mask_of_right_texts = regular_encode(\n        texts=[cur[1] for cur in text_pairs_and_labels],\n        tokenizer=tokenizer, maxlen=maxlen\n    )\n    siamese_labels = np.array([cur[2] for cur in text_pairs_and_labels], dtype=np.int32)\n    left_labels = np.array([cur[3] for cur in text_pairs_and_labels], dtype=np.int32)\n    right_labels = np.array([cur[4] for cur in text_pairs_and_labels], dtype=np.int32)\n    print('Number of positive siamese samples is {0} from {1}.'.format(\n        int(sum(siamese_labels)), siamese_labels.shape[0]))\n    print('Number of positive left samples is {0} from {1}.'.format(\n        int(sum(left_labels)), left_labels.shape[0]))\n    print('Number of positive right samples is {0} from {1}.'.format(\n        int(sum(right_labels)), right_labels.shape[0]))\n    dataset = tf.data.Dataset.from_tensor_slices(\n        (\n            (\n                tokens_of_left_texts, mask_of_left_texts,\n                tokens_of_right_texts, mask_of_right_texts\n            ),\n            (\n                siamese_labels,\n                left_labels,\n                right_labels\n            )\n        )\n    ).batch(batch_size)\n    if for_training:\n        dataset = dataset.repeat()\n    del text_pairs_and_labels\n    return dataset, n_steps","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_feature_extractor(transformer_name: str, hidden_state_size: int,\n                            max_len: int) -> Tuple[tf.keras.Model, tf.keras.Model]:\n    word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                     name=\"base_word_ids\")\n    attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                           name=\"base_attention_mask\")\n    transformer_layer = TFXLMRobertaModel.from_pretrained(\n        pretrained_model_name_or_path=transformer_name,\n        name='Transformer'\n    )\n    sequence_output = transformer_layer([word_ids, attention_mask])[0]\n    output_mask = MaskCalculator(\n        output_dim=hidden_state_size, trainable=False,\n        name='OutMaskCalculator'\n    )(attention_mask)\n    masked_sequence_output = tf.keras.layers.Multiply(\n        name='OutMaskMultiplicator'\n    )([output_mask, sequence_output])\n    masked_sequence_output = tf.keras.layers.Masking(\n        name='OutMasking'\n    )(masked_sequence_output)\n    pooled_output = tf.keras.layers.GlobalAvgPool1D(name='AvePool')(masked_sequence_output)\n    text_embedding = tf.keras.layers.Lambda(\n        lambda x: tf.math.l2_normalize(x, axis=1),\n        name='Emdedding'\n    )(pooled_output)\n    cls_layer = tf.keras.layers.Dropout(rate=0.3, name='ClsDropout')(pooled_output)\n    cls_layer = tf.keras.layers.Dense(\n        units=1, activation='sigmoid', use_bias=True,\n        kernel_initializer=tf.keras.initializers.GlorotNormal(seed=42),\n        kernel_regularizer=tf.keras.regularizers.l2(l=1e-4),\n        name='ClsOutput'\n    )(cls_layer)\n    cls_model = tf.keras.Model(\n        inputs=[word_ids, attention_mask],\n        outputs=cls_layer,\n        name='Classifier'\n    )\n    cls_model.build(input_shape=[(None, max_len), (None, max_len)])\n    fe_model = tf.keras.Model(\n        inputs=[word_ids, attention_mask],\n        outputs=text_embedding,\n        name='FeatureExtractor'\n    )\n    fe_model.build(input_shape=[(None, max_len), (None, max_len)])\n    return cls_model, fe_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def euclidean_distance(vects):\n    x, y = vects\n    sum_square = tf.keras.backend.sum(tf.keras.backend.square(x - y),\n                                      axis=1, keepdims=True)\n    return tf.keras.backend.sqrt(\n        tf.keras.backend.maximum(sum_square, tf.keras.backend.epsilon())\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def eucl_dist_output_shape(shapes):\n    shape1, shape2 = shapes\n    return (shape1[0], 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_siamese_nn(transformer_name: str, hidden_state_size: int, max_len: int,\n                     max_lr: float, base_lr: float, steps_per_epoch: int) -> \\\n        Tuple[tf.keras.Model, tf.keras.Model, tf.keras.Model]:\n    left_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                          name=\"left_word_ids\")\n    left_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                                name=\"left_attention_mask\")\n    right_word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                           name=\"right_word_ids\")\n    right_attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                                 name=\"right_attention_mask\")\n    cls_, fe_ = build_feature_extractor(transformer_name, hidden_state_size, max_len)\n    left_text_embedding = fe_([left_word_ids, left_attention_mask])\n    right_text_embedding = fe_([right_word_ids, right_attention_mask])\n    distance_layer = tf.keras.layers.Lambda(\n        function=euclidean_distance,\n        output_shape=eucl_dist_output_shape,\n        name='L2DistLayer'\n    )([left_text_embedding, right_text_embedding])\n    left_cls_layer = cls_([left_word_ids, left_attention_mask])\n    right_cls_layer = cls_([right_word_ids, right_attention_mask])\n    nn = tf.keras.Model(\n        inputs=[left_word_ids, left_attention_mask, right_word_ids, right_attention_mask],\n        outputs=[distance_layer, left_cls_layer, right_cls_layer],\n        name='SiameseXLMR'\n    )\n    nn.compile(\n        optimizer=tf.keras.optimizers.Adam(\n            learning_rate=tfa.optimizers.Triangular2CyclicalLearningRate(\n                initial_learning_rate=base_lr,\n                maximal_learning_rate=max_lr,\n                step_size=int(round(steps_per_epoch * 0.75))\n            )\n        ),\n        loss=[\n            DBLLogLoss(),\n            tf.keras.losses.BinaryCrossentropy(),\n            tf.keras.losses.BinaryCrossentropy()\n        ],\n        loss_weights=[\n            1.0,\n            0.3,\n            0.3\n        ]\n    )\n    fe_.summary()\n    nn.summary()\n    return nn, cls_, fe_","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_classifier(config: XLMRobertaConfig, hidden_state_size: int, max_len: int,\n                     max_lr: float, base_lr: float, steps_per_epoch: int,\n                     language: str) -> tf.keras.Model:\n    word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                     name=\"word_ids_{0}\".format(language))\n    attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                           name=\"attention_mask_{0}\".format(language))\n    transformer_layer = TFXLMRobertaModel(\n        config=config,\n        name='Transformer_{0}'.format(language.title())\n    )\n    sequence_output = transformer_layer([word_ids, attention_mask])[0]\n    output_mask = MaskCalculator(\n        output_dim=hidden_state_size, trainable=False,\n        name='OutMaskCalculator_{0}'.format(language)\n    )(attention_mask)\n    masked_sequence_output = tf.keras.layers.Multiply(\n        name='OutMaskMultiplicator_{0}'.format(language)\n    )([output_mask, sequence_output])\n    masked_sequence_output = tf.keras.layers.Masking(\n        name='OutMasking_{0}'.format(language)\n    )(masked_sequence_output)\n    pooled_output = tf.keras.layers.GlobalAvgPool1D(\n        name='AvePool_{0}'.format(language)\n    )(masked_sequence_output)\n    cls_layer = tf.keras.layers.Dropout(rate=0.3, name='ClsDropout')(pooled_output)\n    cls_layer = tf.keras.layers.Dense(\n        units=1, activation='sigmoid', use_bias=True,\n        kernel_initializer=tf.keras.initializers.GlorotNormal(seed=42),\n        kernel_regularizer=tf.keras.regularizers.l2(l=1e-4),\n        name='ClsOutput_{0}'.format(language.title())\n    )(cls_layer)\n    cls_model = tf.keras.Model(\n        inputs=[word_ids, attention_mask],\n        outputs=cls_layer,\n        name='Classifier_{0}'.format(language.title())\n    )\n    cls_model.compile(\n        optimizer=tf.keras.optimizers.Adam(\n            learning_rate=tfa.optimizers.Triangular2CyclicalLearningRate(\n                initial_learning_rate=base_lr,\n                maximal_learning_rate=max_lr,\n                step_size=int(round(steps_per_epoch * 2.0))\n            )\n        ),\n        loss=DiceLoss(),\n        metrics=[tf.keras.metrics.AUC()]\n    )\n    cls_model.summary()\n    return cls_model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_training_process(history: tf.keras.callbacks.History, metric_name: str,\n                          figure_id: int=1):\n    val_metric_name = 'val_' + metric_name\n    err_msg = 'The metric \"{0}\" is not found! Available metrics are: {1}'.format(\n        metric_name, list(history.history.keys()))\n    assert val_metric_name in history.history, err_msg\n    err_msg = 'The metric \"{0}\" is not found! Available metrics are: {1}'.format(\n        val_metric_name, list(history.history.keys()))\n    assert metric_name in history.history, err_msg\n    assert len(history.history[metric_name]) == len(history.history['val_' + metric_name])\n    plt.figure(figure_id)\n    plt.plot(list(range(len(history.history[metric_name]))),\n             history.history[metric_name], label='Training {0}'.format(metric_name))\n    plt.plot(list(range(len(history.history['val_' + metric_name]))),\n             history.history['val_' + metric_name], label='Validation {0}'.format(metric_name))\n    plt.xlabel('Epochs')\n    plt.ylabel(metric_name)\n    plt.title('Training process')\n    plt.legend(loc='best')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def train_siamese_nn(nn: tf.keras.Model, trainset: tf.data.Dataset, steps_per_trainset: int,\n                     validset: tf.data.Dataset, steps_per_validset: int,\n                     max_duration: int, model_weights_path: str):\n    assert steps_per_validset >= 2\n    assert steps_per_trainset >= 10 * steps_per_validset\n    steps_per_epoch = 10 * steps_per_validset\n    n_epochs = (3 * steps_per_trainset) // steps_per_epoch\n    print('Maximal duration of the Siamese NN training is {0} seconds.'.format(max_duration))\n    print('n_epochs = {0}, steps_per_epoch = {1}'.format(n_epochs, steps_per_epoch))\n    print('steps_per_trainset = {0}, steps_per_validset = {1}'.format(\n        steps_per_trainset, steps_per_validset\n    ))\n    callbacks = [\n        tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True),\n        tf.keras.callbacks.ModelCheckpoint(model_weights_path, monitor='val_loss',\n                                           mode='min', save_best_only=True,\n                                           save_weights_only=True, verbose=True)\n    ]\n    history = nn.fit(\n        trainset,\n        steps_per_epoch=steps_per_epoch,\n        validation_data=validset,\n        epochs=n_epochs, callbacks=callbacks\n    )\n    show_training_process(history, 'loss')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def show_roc_auc(y_true: np.ndarray, probabilities: np.ndarray, label: str,\n                 figure_id: int=1):\n    plt.figure(figure_id)\n    plt.plot([0, 1], [0, 1], 'k--')\n    print('ROC-AUC score for {0} is {1:.9f}'.format(\n        label, roc_auc_score(y_true=y_true, y_score=probabilities)\n    ))\n    fpr, tpr, _ = roc_curve(y_true=y_true, y_score=probabilities)\n    plt.plot(fpr, tpr, label=label)\n    plt.xlabel('False positive rate')\n    plt.ylabel('True positive rate')\n    plt.title('ROC curve')\n    plt.legend(loc='best')\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def calculate_features_of_texts(texts: Dict[str, List[Tuple[str, int]]],\n                                tokenizer: XLMRobertaTokenizer, maxlen: int,\n                                fe: tf.keras.Model, batch_size: int,\n                                max_dataset_size: int = 0) -> \\\n        Dict[str, Tuple[np.ndarray, np.ndarray]]:\n    languages = sorted(list(texts.keys()))\n    datasets_by_languages = dict()\n    if max_dataset_size > 0:\n        max_size_per_lang = max_dataset_size // len(languages)\n        err_msg = '{0} is too small number of dataset samples!'.format(max_dataset_size)\n        assert max_size_per_lang > 0, err_msg\n    else:\n        max_size_per_lang = 0\n    for cur_lang in languages:\n        selected_indices = list(range(len(texts[cur_lang])))\n        if max_size_per_lang > 0:\n            if len(selected_indices) > max_size_per_lang:\n                selected_indices = random.sample(\n                    population=selected_indices,\n                    k=max_size_per_lang\n                )\n        tokens_of_texts, mask_of_texts = regular_encode(\n            texts=[texts[cur_lang][idx][0] for idx in selected_indices],\n            tokenizer=tokenizer, maxlen=maxlen\n        )\n        X = []\n        n_batches = int(np.ceil(len(selected_indices) / float(batch_size)))\n        for batch_idx in range(n_batches):\n            batch_start = batch_idx * batch_size\n            batch_end = min(len(selected_indices), batch_start + batch_size)\n            res = fe.predict_on_batch(\n                [\n                    tokens_of_texts[batch_start:batch_end],\n                    mask_of_texts[batch_start:batch_end]\n                ]\n            )\n            if not isinstance(res, np.ndarray):\n                res = res.numpy()\n            X.append(res)\n            del res\n        X = np.vstack(X)\n        y = np.array([texts[cur_lang][idx][1] for idx in selected_indices], dtype=np.int32)\n        datasets_by_languages[cur_lang] = (X, y)\n        del X, y, selected_indices\n    return datasets_by_languages","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def predict_with_model(classifier: tf.keras.Model,\n                       input_data: Tuple[np.ndarray, np.ndarray],\n                       batch_size: int) -> np.ndarray:\n    predicted = []\n    n_batches = int(np.ceil(input_data[0].shape[0] / float(batch_size)))\n    for batch_idx in range(n_batches):\n        batch_start = batch_idx * batch_size\n        batch_end = min(batch_start + batch_size, input_data[0].shape[0])\n        res = classifier.predict_on_batch(\n            (\n                input_data[0][batch_start:batch_end],\n                input_data[1][batch_start:batch_end]\n            )\n        )\n        if not isinstance(res, np.ndarray):\n            res = res.numpy()\n        predicted.append(res.flatten())\n        del res\n    return np.concatenate(predicted)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def do_submit(data_for_training: Dict[str, Tuple[Tuple[np.ndarray, np.ndarray], np.ndarray]],\n              data_for_submission: Tuple[np.ndarray, np.ndarray],\n              language_for_validation: str, maxlen: int, base_classifier: tf.keras.Model,\n              max_duration: int, batch_size: int) -> Tuple[np.ndarray, int]:\n    assert language_for_validation in data_for_training\n    print('Duration of the training procedure must be less ' \\\n          'than {0} seconds.'.format(max_duration))\n    print('')\n    start_time = time.time()\n    X_train1 = []\n    X_train2 = []\n    y_train = []\n    X_val1 = []\n    X_val2 = []\n    y_val = []\n    for cur_lang in sorted(list(data_for_training.keys())):\n        if cur_lang == language_for_validation:\n            X_val1.append(data_for_training[cur_lang][0][0])\n            X_val2.append(data_for_training[cur_lang][0][1])\n            y_val.append(data_for_training[cur_lang][1])\n        else:\n            X_train1.append(data_for_training[cur_lang][0][0])\n            X_train2.append(data_for_training[cur_lang][0][1])\n            y_train.append(data_for_training[cur_lang][1])\n    X_train = (np.vstack(X_train1), np.vstack(X_train2))\n    del X_train1, X_train2\n    y_train = np.concatenate(y_train)\n    trainset = tf.data.Dataset.from_tensor_slices(\n        (X_train, y_train)\n    ).repeat().batch(batch_size)\n    del X_train\n    print('Labeled dataset for training contains {0} samples.'.format(y_train.shape[0]))\n    print('Number of positive samples is {0} from {1}.'.format(\n        int(sum(y_train)), y_train.shape[0]))\n    print('')\n    steps_per_epoch = int(np.ceil(y_train.shape[0] / float(batch_size)))\n    del y_train\n    X_val = (np.vstack(X_val1), np.vstack(X_val2))\n    del X_val1, X_val2\n    y_val = np.concatenate(y_val)\n    validset = tf.data.Dataset.from_tensor_slices(\n        (X_val, y_val)\n    ).batch(batch_size)\n    print('Dataset for validation contains {0} samples.'.format(y_val.shape[0]))\n    print('Number of positive samples is {0} from {1}.'.format(int(sum(y_val)),\n                                                               y_val.shape[0]))\n    print('')\n    steps_per_validset = int(np.ceil(y_val.shape[0] / float(batch_size)))\n    if steps_per_epoch <= (3 * steps_per_validset):\n        n_epochs = 25\n    else:\n        n_epochs = (25 * steps_per_epoch) // (3 * steps_per_validset)\n        steps_per_epoch = 3 * steps_per_validset\n    callbacks = [\n        tf.keras.callbacks.EarlyStopping(patience=5, monitor='val_auc', mode='max',\n                                         restore_best_weights=True, verbose=True),\n        tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True)\n    ]\n    print('n_epochs = {0}, steps_per_epoch = {1}, val_steps = {2}'.format(\n        n_epochs, steps_per_epoch, steps_per_validset))\n    history = base_classifier.fit(\n        trainset,\n        steps_per_epoch=steps_per_epoch,\n        validation_data=validset,\n        epochs=n_epochs, callbacks=callbacks\n    )\n    postprocessing_start_time = time.time()\n    show_training_process(history, 'auc', figure_id=1)\n    del history, trainset, validset\n    print('')\n    probabilities = predict_with_model(\n        classifier=base_classifier,\n        input_data=X_val,\n        batch_size=batch_size\n    )\n    show_roc_auc(y_true=y_val, probabilities=probabilities,\n                 label='language \"{0}\"'.format(language_for_validation),\n                 figure_id=2)\n    del probabilities, y_val, X_val\n    gc.collect()\n    probabilities = predict_with_model(\n        classifier=base_classifier,\n        input_data=data_for_submission,\n        batch_size=batch_size\n    )\n    assert probabilities.shape[0] == len(identifiers_for_submission)\n    gc.collect()\n    end_time = time.time()\n    print('Duration of the final submission for language '\\\n          '\"{0}\" is {1:.3f} seconds'.format(\n        language_for_validation, end_time - start_time))\n    print('Postprocessing duration (after training) is ' \\\n          '{0:.3f} seconds.'.format(end_time - postprocessing_start_time))\n    return probabilities, int(np.ceil(end_time - postprocessing_start_time))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"experiment_start_time = time.time()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    model_name = 'jplu/tf-xlm-roberta-large'\n    max_seq_len = 256\n    batch_size_for_siamese = 4 * strategy.num_replicas_in_sync\n    batch_size_for_cls = 8 * strategy.num_replicas_in_sync\nelse:\n    strategy = tf.distribute.get_strategy()\n    if strategy.num_replicas_in_sync == 1:\n        physical_devices = tf.config.list_physical_devices('GPU') \n        tf.config.experimental.set_memory_growth(physical_devices[0], True)\n    max_seq_len = 128\n    model_name = 'jplu/tf-xlm-roberta-base'\n    batch_size_for_siamese = 8 * strategy.num_replicas_in_sync\n    batch_size_for_cls = 16 * strategy.num_replicas_in_sync\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\nprint('Model name: {0}'.format(model_name))\nprint('Maximal length of sequence is {0}'.format(max_seq_len))\nprint('Batch size for Siamese NN is {0}'.format(batch_size_for_siamese))\nprint('Batch size for classifier is {0}'.format(batch_size_for_cls))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"random.seed(42)\nnp.random.seed(42)\ntf.random.set_seed(42)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"siamese_learning_rate_max = 1e-5\nsiamese_learning_rate_min = 5e-6\nclassifier_learning_rate_max = 2e-6\nclassifier_learning_rate_min = 6e-7\ndataset_dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification'\ntmp_roberta_name = '/kaggle/working/base_nn.h5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"xlmroberta_tokenizer = AutoTokenizer.from_pretrained(model_name)\nxlmroberta_config = XLMRobertaConfig.from_pretrained(model_name)\nprint(xlmroberta_config)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sentence_embedding_size = xlmroberta_config.hidden_size\nprint('Sentence embedding size is {0}'.format(sentence_embedding_size))\nassert max_seq_len <= xlmroberta_config.max_position_embeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corpus_for_training = load_train_set(\n    os.path.join(dataset_dir, \"jigsaw-unintended-bias-train.csv\"),\n    text_field=\"comment_text\", lang_field=\"lang\",\n    sentiment_fields=[\"toxic\", \"severe_toxicity\", \"obscene\", \"identity_attack\", \"insult\",\n                      \"threat\"]\n)\nassert 'en' in corpus_for_training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"corpus_for_validation = load_train_set(\n    os.path.join(dataset_dir, \"jigsaw-toxic-comment-train.csv\"),\n    text_field=\"comment_text\", lang_field=\"lang\",\n    sentiment_fields=[\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\",\n                      \"identity_hate\"]\n)\nassert 'en' in corpus_for_validation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"multilingual_corpus = load_train_set(\n    os.path.join(dataset_dir, \"validation.csv\"),\n    text_field=\"comment_text\", lang_field=\"lang\", sentiment_fields=[\"toxic\", ]\n)\nassert 'en' not in multilingual_corpus\nmax_size = 0\nprint('Multilingual data:')\nfor language in sorted(list(multilingual_corpus.keys())):\n    print('  {0}\\t\\t{1} samples'.format(language, len(multilingual_corpus[language])))\n    assert set(map(lambda cur: cur[1], multilingual_corpus[language])) == {0, 1}\n    if len(multilingual_corpus[language]) > max_size:\n        max_size = len(multilingual_corpus[language])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"texts_for_submission = load_test_set(\n    os.path.join(dataset_dir, \"test.csv\"),\n    text_field=\"content\", lang_field=\"lang\", id_field=\"id\"\n)\nprint('Data for submission:')\nfor language in sorted(list(texts_for_submission.keys())):\n    print('  {0}\\t\\t{1} samples'.format(language, len(texts_for_submission[language])))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_for_training, n_train_batches = build_siamese_dataset(\n    texts=corpus_for_training, dataset_size=300000,\n    tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,\n    batch_size=batch_size_for_siamese, for_training=True\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del corpus_for_training","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_for_validation, n_val_batches = build_siamese_dataset(\n    texts=multilingual_corpus, dataset_size=2000,\n    tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,\n    batch_size=batch_size_for_siamese, for_training=False\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"multilingual_corpus['en'] = random.sample(\n    population=corpus_for_validation['en'],\n    k=max_size\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del corpus_for_validation","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preparing_duration = int(round(time.time() - experiment_start_time))\nprint(\"Duration of data loading and preparing to the Siamese NN training is \"\n      \"{0} seconds.\".format(preparing_duration))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    siamese_network, neural_classifier, feature_extractor = build_siamese_nn(\n        transformer_name=model_name,\n        hidden_state_size=sentence_embedding_size,\n        max_len=max_seq_len,\n        max_lr=siamese_learning_rate_max,\n        base_lr=siamese_learning_rate_min,\n        steps_per_epoch=n_val_batches * 10\n    )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_siamese_nn(nn=siamese_network,\n                 trainset=dataset_for_training, steps_per_trainset=n_train_batches,\n                 validset=dataset_for_validation, steps_per_validset=n_val_batches,\n                 max_duration=int(round(3600 * 0.7 - preparing_duration)),\n                 model_weights_path=tmp_roberta_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del dataset_for_training\ndel dataset_for_validation\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"siamese_network.load_weights(tmp_roberta_name)\nneural_classifier.save_weights(tmp_roberta_name, overwrite=True, save_format='h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del neural_classifier, siamese_network","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_for_training = calculate_features_of_texts(\n    texts=multilingual_corpus,\n    tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,\n    fe=feature_extractor,\n    batch_size=batch_size_for_siamese,\n    max_dataset_size=min(600, batch_size_for_siamese * 9)\n)\nassert len(dataset_for_training) == 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dataset_for_submission = calculate_features_of_texts(\n    texts=texts_for_submission,\n    tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len,\n    fe=feature_extractor,\n    batch_size=batch_size_for_siamese,\n    max_dataset_size=min(300, batch_size_for_siamese * 3)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_embedded = []\ny_embedded = []\nfor cur_lang in dataset_for_training:\n    X_embedded.append(dataset_for_training[cur_lang][0])\n    y_embedded.append(dataset_for_training[cur_lang][1])\nfor cur_lang in dataset_for_submission:\n    X_embedded.append(dataset_for_submission[cur_lang][0])\n    y_embedded.append(\n        np.array(\n            [-1 for _ in range(dataset_for_submission[cur_lang][0].shape[0])],\n            dtype=np.int32\n        )\n    )\nX_embedded = np.vstack(X_embedded)\ny_embedded = np.concatenate(y_embedded)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del dataset_for_training, dataset_for_submission, feature_extractor","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_embedded = TSNE(n_components=2, n_jobs=-1).fit_transform(X_embedded)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"indices_of_unknown_classes = list(filter(\n    lambda sample_idx: y_embedded[sample_idx] < 0,\n    range(len(y_embedded))\n))\nxy = X_embedded[indices_of_unknown_classes]\nplt.plot(xy[:, 0], xy[:, 1], 'o', color='b', markersize=2,\n         label='Unlabeled texts for submission')\nindices_of_negative_classes = list(filter(\n    lambda sample_idx: y_embedded[sample_idx] == 0,\n    range(len(y_embedded))\n))\nxy = X_embedded[indices_of_negative_classes]\nplt.plot(xy[:, 0], xy[:, 1], 'o', color='g', markersize=4,\n         label='Normal texts')\nindices_of_positive_classes = list(filter(\n    lambda sample_idx: y_embedded[sample_idx] > 0,\n    range(len(y_embedded))\n))\nxy = X_embedded[indices_of_positive_classes]\nplt.plot(xy[:, 0], xy[:, 1], 'o', color='r', markersize=6,\n         label='Toxic texts')\nplt.title('Toxic and normal texts')\nplt.legend(loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del indices_of_negative_classes\ndel indices_of_positive_classes\ndel indices_of_unknown_classes\ndel X_embedded, y_embedded","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()\ntf.keras.backend.clear_session()\nif tpu:\n    tf.tpu.experimental.shutdown_tpu_system(tpu)\n    del strategy\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"all_languages_for_training = sorted(list(multilingual_corpus.keys()))\nprint('Languages in the labeled trainset: ' \\\n      '{0}'.format(all_languages_for_training))\nassert len(all_languages_for_training) == 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"training_set_size = 0\nfor cur_lang in all_languages_for_training:\n    multilingual_corpus[cur_lang] = (\n        regular_encode(\n            texts=[cur[0] for cur in multilingual_corpus[cur_lang]],\n            tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len\n        ),\n        np.array(\n            [cur[1] for cur in multilingual_corpus[cur_lang]],\n            dtype=np.int32\n        )\n    )\n    training_set_size += multilingual_corpus[cur_lang][1].shape[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"texts_for_submission_ = []\nidentifiers_for_submission = []\nfor cur_lang in sorted(list(texts_for_submission.keys())):\n    texts_for_submission_ += [\n        cur[0] for cur in texts_for_submission[cur_lang]\n    ]\n    identifiers_for_submission += [\n        cur[1] for cur in texts_for_submission[cur_lang]\n    ]\nX_submit = regular_encode(\n    texts=texts_for_submission_,\n    tokenizer=xlmroberta_tokenizer, maxlen=max_seq_len\n)\nidentifiers_for_submission = np.array(\n    identifiers_for_submission,\n    dtype=np.int32\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del xlmroberta_tokenizer, texts_for_submission_\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"experiment_duration = int(round(time.time() - experiment_start_time))\nprint('Duration of the Siamese XLM-RoBERTa preparing is {0} seconds.'.format(\n    experiment_duration))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    neural_classifier = build_classifier(\n        config=xlmroberta_config,\n        hidden_state_size=sentence_embedding_size,\n        max_len=max_seq_len,\n        max_lr=classifier_learning_rate_max,\n        base_lr=classifier_learning_rate_min,\n        steps_per_epoch=training_set_size // batch_size_for_cls,\n        language=all_languages_for_training[0]\n    )\nneural_classifier.load_weights(tmp_roberta_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_of_submission1, postprocessing_duration = do_submit(\n    data_for_training=multilingual_corpus,\n    data_for_submission=X_submit,\n    language_for_validation=all_languages_for_training[0],\n    maxlen=max_seq_len, base_classifier=neural_classifier,\n    batch_size=batch_size_for_cls,\n    max_duration=int(round(2.7 * 3600) - (time.time() - experiment_start_time)) // 5\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert result_of_submission1.shape == identifiers_for_submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del neural_classifier\ngc.collect()\ntf.keras.backend.clear_session()\nif tpu:\n    tf.tpu.experimental.shutdown_tpu_system(tpu)\n    del strategy\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    neural_classifier = build_classifier(\n        config=xlmroberta_config,\n        hidden_state_size=sentence_embedding_size,\n        max_len=max_seq_len,\n        max_lr=classifier_learning_rate_max,\n        base_lr=classifier_learning_rate_min,\n        steps_per_epoch=training_set_size // batch_size_for_cls,\n        language=all_languages_for_training[1]\n    )\nneural_classifier.load_weights(tmp_roberta_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_of_submission2, _ = do_submit(\n    data_for_training=multilingual_corpus,\n    data_for_submission=X_submit,\n    language_for_validation=all_languages_for_training[1],\n    maxlen=max_seq_len, base_classifier=neural_classifier,\n    batch_size=batch_size_for_cls,\n    max_duration=(int(round(2.7 * 3600) - (time.time() - experiment_start_time)) // 3 - \n                  postprocessing_duration)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert result_of_submission2.shape == identifiers_for_submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del neural_classifier\ngc.collect()\ntf.keras.backend.clear_session()\nif tpu:\n    tf.tpu.experimental.shutdown_tpu_system(tpu)\n    del strategy\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    neural_classifier = build_classifier(\n        config=xlmroberta_config,\n        hidden_state_size=sentence_embedding_size,\n        max_len=max_seq_len,\n        max_lr=classifier_learning_rate_max,\n        base_lr=classifier_learning_rate_min,\n        steps_per_epoch=training_set_size // batch_size_for_cls,\n        language=all_languages_for_training[2]\n    )\nneural_classifier.load_weights(tmp_roberta_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_of_submission3, _ = do_submit(\n    data_for_training=multilingual_corpus,\n    data_for_submission=X_submit,\n    language_for_validation=all_languages_for_training[2],\n    maxlen=max_seq_len, base_classifier=neural_classifier,\n    batch_size=batch_size_for_cls,\n    max_duration=(int(round(2.7 * 3600) - (time.time() - experiment_start_time)) // 2 - \n                  postprocessing_duration)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert result_of_submission3.shape == identifiers_for_submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del neural_classifier\ngc.collect()\ntf.keras.backend.clear_session()\nif tpu:\n    tf.tpu.experimental.shutdown_tpu_system(tpu)\n    del strategy\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    neural_classifier = build_classifier(\n        config=xlmroberta_config,\n        hidden_state_size=sentence_embedding_size,\n        max_len=max_seq_len,\n        max_lr=classifier_learning_rate_max,\n        base_lr=classifier_learning_rate_min,\n        steps_per_epoch=training_set_size // batch_size_for_cls,\n        language=all_languages_for_training[3]\n    )\nneural_classifier.load_weights(tmp_roberta_name)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"result_of_submission4, _ = do_submit(\n    data_for_training=multilingual_corpus,\n    data_for_submission=X_submit,\n    language_for_validation=all_languages_for_training[3],\n    maxlen=max_seq_len, base_classifier=neural_classifier,\n    batch_size=batch_size_for_cls,\n    max_duration=(int(round(2.7 * 3600) - (time.time() - experiment_start_time)) - \n                  postprocessing_duration)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"assert result_of_submission4.shape == identifiers_for_submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with codecs.open('submission.csv', mode='w', encoding='utf-8', errors='ignore') as fp:\n    fp.write('id,toxic\\n')\n    for sample_idx in range(identifiers_for_submission.shape[0]):\n        proba_val = result_of_submission1[sample_idx]\n        proba_val += result_of_submission2[sample_idx]\n        proba_val += result_of_submission3[sample_idx]\n        proba_val += result_of_submission4[sample_idx]\n        proba_val /= 4.0\n        id_val = identifiers_for_submission[sample_idx]\n        fp.write('{0},{1:.9f}\\n'.format(id_val, proba_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Experiment duration is {0:.3f}.'.format(time.time() - experiment_start_time))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}