{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"import codecs\nimport copy\nimport csv\nimport gc\nimport os\nimport random\nimport time\nfrom typing import Dict, List, Tuple","metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport numpy as np\nfrom sklearn.metrics import roc_auc_score, roc_curve\nimport tensorflow as tf\nimport tensorflow_addons as tfa\nfrom transformers import TFBertModel, BertConfig\nfrom transformers import AutoTokenizer, BertTokenizer","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def generate_random_seed() -> int:\n    return random.randint(0, 2147483648)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def regular_encode(texts: List[str], tokenizer: BertTokenizer,\n                   maxlen: int) -> Tuple[np.ndarray, np.ndarray]:\n    err_msg = '\"{0}\" is wrong type for the text list!'.format(type(texts))\n    assert isinstance(texts, list) or isinstance(texts, tuple), err_msg\n    enc_di = tokenizer.batch_encode_plus(\n        texts,\n        return_token_type_ids=False,\n        padding='max_length',\n        max_length=maxlen\n    )\n    err_msg = '{0} != {1}'.format(len(texts), len(enc_di['input_ids']))\n    assert len(texts) == len(enc_di['input_ids']), err_msg\n    err_msg = '{0} != {1}'.format(len(texts), len(enc_di['attention_mask']))\n    assert len(texts) == len(enc_di['attention_mask']), err_msg\n    encoded_tokens = np.zeros((len(texts), maxlen), dtype=np.int32)\n    encoded_masks = np.zeros((len(texts), maxlen), dtype=np.int32)\n    for sample_idx, (encoded_cur_text, encoded_cur_mask) in enumerate(\n        zip(enc_di['input_ids'], enc_di['attention_mask'])\n    ):\n        n_text = len(encoded_cur_text)\n        n_mask = len(encoded_cur_mask)\n        err_msg = 'Tokens and masks of texts \"{0}\" are different! '\\\n                  '{1} != {2}'.format(texts[sample_idx], n_text, n_mask)\n        assert n_text == n_mask, err_msg\n        if n_text >= maxlen:\n            encoded_tokens[sample_idx] = np.array(encoded_cur_text[0:maxlen],\n                                                  dtype=np.int32)\n            encoded_masks[sample_idx] = np.array(encoded_cur_mask[0:maxlen],\n                                                 dtype=np.int32)\n        else:\n            padding = [0 for _ in range(maxlen - n_text)]\n            encoded_tokens[sample_idx] = np.array(encoded_cur_text + padding,\n                                                  dtype=np.int32)\n            encoded_masks[sample_idx] = np.array(encoded_cur_mask + padding,\n                                                 dtype=np.int32)\n    return encoded_tokens, encoded_masks","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_train_set(file_name: str, text_field: str, sentiment_fields: List[str],\n                   lang_field: str) -> Dict[str, List[Tuple[str, int]]]:\n    assert len(sentiment_fields) > 0, 'List of sentiment fields is empty!'\n    header = []\n    line_idx = 1\n    data_by_lang = dict()\n    with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore') as fp:\n        data_reader = csv.reader(fp, quotechar='\"', delimiter=',')\n        for row in data_reader:\n            if len(row) > 0:\n                err_msg = 'File \"{0}\": line {1} is wrong!'.format(file_name, line_idx)\n                if len(header) == 0:\n                    header = copy.copy(row)\n                    err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(text_field)\n                    assert text_field in header, err_msg2\n                    for cur_field in sentiment_fields:\n                        err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(\n                            cur_field)\n                        assert cur_field in header, err_msg2\n                    text_field_index = header.index(text_field)\n                    try:\n                        lang_field_index = header.index(lang_field)\n                    except:\n                        lang_field_index = -1\n                    indices_of_sentiment_fields = []\n                    for cur_field in sentiment_fields:\n                        indices_of_sentiment_fields.append(header.index(cur_field))\n                else:\n                    if len(row) == len(header):\n                        text = row[text_field_index].strip()\n                        assert len(text) > 0, err_msg + ' Text is empty!'\n                        if lang_field_index >= 0:\n                            cur_lang = row[lang_field_index].strip()\n                            assert len(cur_lang) > 0, err_msg + ' Language is empty!'\n                        else:\n                            cur_lang = 'en'\n                        max_proba = 0.0\n                        for cur_field_idx in indices_of_sentiment_fields:\n                            try:\n                                cur_proba = float(row[cur_field_idx])\n                            except:\n                                cur_proba = -1.0\n                            err_msg2 = err_msg + ' Value {0} is wrong!'.format(\n                                row[cur_field_idx]\n                            )\n                            assert (cur_proba >= 0.0) and (cur_proba <= 1.0), err_msg2\n                            if cur_proba > max_proba:\n                                max_proba = cur_proba\n                        new_label = 1 if max_proba >= 0.5 else 0\n                        if cur_lang not in data_by_lang:\n                            data_by_lang[cur_lang] = []\n                        data_by_lang[cur_lang].append((text, new_label))\n            if line_idx % 10000 == 0:\n                print('{0} lines of the \"{1}\" have been processed...'.format(line_idx,\n                                                                             file_name))\n            line_idx += 1\n    if line_idx > 0:\n        if (line_idx - 1) % 10000 != 0:\n            print('{0} lines of the \"{1}\" have been processed...'.format(line_idx - 1,\n                                                                         file_name))\n    return data_by_lang","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def load_test_set(file_name: str, id_field: str, text_field: str,\n                  lang_field: str) -> Dict[str, List[Tuple[str, int]]]:\n    header = []\n    line_idx = 1\n    data_by_lang = dict()\n    with codecs.open(file_name, mode='r', encoding='utf-8', errors='ignore') as fp:\n        data_reader = csv.reader(fp, quotechar='\"', delimiter=',')\n        for row in data_reader:\n            if len(row) > 0:\n                err_msg = 'File \"{0}\": line {1} is wrong!'.format(file_name, line_idx)\n                if len(header) == 0:\n                    header = copy.copy(row)\n                    err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(text_field)\n                    assert text_field in header, err_msg2\n                    err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(id_field)\n                    assert id_field in header, err_msg2\n                    err_msg2 = err_msg + ' Field \"{0}\" is not found!'.format(lang_field)\n                    assert lang_field in header, err_msg2\n                    id_field_index = header.index(id_field)\n                    text_field_index = header.index(text_field)\n                    lang_field_index = header.index(lang_field)\n                else:\n                    if len(row) == len(header):\n                        try:\n                            id_value = int(row[id_field_index])\n                        except:\n                            id_value = -1\n                        err_msg2 = err_msg + ' {0} is wrong ID!'.format(\n                            row[id_field_index])\n                        assert id_value >= 0, err_msg2\n                        text = row[text_field_index].strip()\n                        assert len(text) > 0, err_msg + ' Text is empty!'\n                        if lang_field_index >= 0:\n                            cur_lang = row[lang_field_index].strip()\n                            assert len(cur_lang) > 0, err_msg + ' Language is empty!'\n                        else:\n                            cur_lang = 'en'\n                        if cur_lang not in data_by_lang:\n                            data_by_lang[cur_lang] = []\n                        data_by_lang[cur_lang].append((text, id_value))\n            if line_idx % 10000 == 0:\n                print('{0} lines of the \"{1}\" have been processed...'.format(line_idx,\n                                                                             file_name))\n            line_idx += 1\n    if line_idx > 0:\n        if (line_idx - 1) % 10000 != 0:\n            print('{0} lines of the \"{1}\" have been processed...'.format(line_idx - 1,\n                                                                         file_name))\n    return data_by_lang","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_dataset(texts: Dict[str, List[Tuple[str, int]]],\n                  dataset_size: int, tokenizer: BertTokenizer,\n                  maxlen: int, batch_size: int,\n                  shuffle: bool) -> Tuple[tf.data.Dataset, int]:\n    texts_and_labels = []\n    dataset_size_by_lang = int(round(dataset_size / float(len(texts))))\n    for lang in texts:\n        print('{0}:'.format(lang))\n        n_lang = 0\n        if shuffle:\n            if len(texts[lang]) > dataset_size_by_lang:\n                texts_and_labels += random.sample(texts[lang], k=dataset_size_by_lang)\n                n_lang += dataset_size_by_lang\n            elif len(texts[lang]) < dataset_size_by_lang:\n                texts_and_labels += texts[lang]\n                n_lang += len(texts[lang])\n                n = dataset_size_by_lang - len(texts[lang])\n                while n >= len(texts[lang]):\n                    texts_and_labels += texts[lang]\n                    n -= len(texts[lang])\n                    n_lang += len(texts[lang])\n                if n > 0:\n                    texts_and_labels += random.sample(texts[lang], k=n)\n                    n_lang += n\n            else:\n                texts_and_labels += texts[lang]\n                n_lang += len(texts[lang])\n        else:\n            texts_and_labels += texts[lang]\n            n_lang += len(texts[lang])\n        print('  number of samples is {0};'.format(n_lang))\n    random.shuffle(texts_and_labels)\n    n_steps = len(texts_and_labels) // batch_size\n    print('Samples number of the data set is {0}.'.format(len(texts_and_labels)))\n    tokens_of_texts, mask_of_texts = regular_encode(\n        texts=[cur[0] for cur in texts_and_labels],\n        tokenizer=tokenizer, maxlen=maxlen\n    )\n    toxicity_labels = np.array([cur[1] for cur in texts_and_labels], dtype=np.int32)\n    print('Number of positive siamese samples is {0} from {1}.'.format(\n        int(sum(toxicity_labels)), toxicity_labels.shape[0]))\n    if shuffle:\n        err_msg = '{0} is too small number of samples for the data set!'.format(\n            len(texts_and_labels))\n        assert n_steps >= 50, err_msg\n        dataset = tf.data.Dataset.from_tensor_slices(\n            (\n                (\n                    tokens_of_texts, mask_of_texts\n                ),\n                toxicity_labels\n            )\n        ).repeat().batch(batch_size)\n    else:\n        dataset = tf.data.Dataset.from_tensor_slices(\n            (\n                (\n                    tokens_of_texts, mask_of_texts\n                ),\n                toxicity_labels\n            )\n        ).batch(batch_size)\n    del texts_and_labels\n    return dataset, n_steps","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def build_classifier(transformer_name: str, hidden_state_size: int,\n                     max_len: int, lr: float) -> tf.keras.Model:\n    word_ids = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                     name=\"base_word_ids\")\n    attention_mask = tf.keras.layers.Input(shape=(max_len,), dtype=tf.int32,\n                                           name=\"base_attention_mask\")\n    transformer_layer = TFBertModel.from_pretrained(\n        pretrained_model_name_or_path=transformer_name,\n        name='Transformer'\n    )\n    sequence_output = transformer_layer([word_ids, attention_mask])[0]\n    pooled_output = tf.keras.layers.GlobalAvgPool1D(\n        name='AvePool'\n    )(sequence_output, mask=attention_mask)\n    kernel_init = tf.keras.initializers.GlorotNormal(seed=generate_random_seed())\n    bias_init = tf.keras.initializers.Constant(value=0.0)\n    cls_layer = tf.keras.layers.Dense(\n        units=1, activation='sigmoid',\n        kernel_initializer=kernel_init,\n        bias_initializer=bias_init,\n        name='OutputLayer'\n    )(pooled_output)\n    cls_model = tf.keras.Model(\n        inputs=[word_ids, attention_mask],\n        outputs=cls_layer,\n        name='ToxicityClassifier'\n    )\n    cls_model.compile(\n        optimizer=tfa.optimizers.AdamW(learning_rate=lr, weight_decay=1e-5),\n        loss='binary_crossentropy'\n    )\n    return cls_model","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_training_process(history: tf.keras.callbacks.History, metric_name: str,\n                          figure_id: int=1):\n    val_metric_name = 'val_' + metric_name\n    err_msg = 'The metric \"{0}\" is not found! Available metrics are: {1}'.format(\n        metric_name, list(history.history.keys()))\n    assert metric_name in history.history, err_msg\n    plt.figure(figure_id, figsize=(5, 5))\n    plt.plot(list(range(len(history.history[metric_name]))),\n             history.history[metric_name], label='Training {0}'.format(metric_name))\n    if val_metric_name in history.history:\n        assert len(history.history[metric_name]) == len(history.history['val_' + metric_name])\n        plt.plot(list(range(len(history.history['val_' + metric_name]))),\n                 history.history['val_' + metric_name], label='Validation {0}'.format(metric_name))\n    plt.xlabel('Epochs')\n    plt.ylabel(metric_name)\n    plt.title('Training process')\n    plt.legend(loc='best')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def train_classifier(nn: tf.keras.Model, trainset: tf.data.Dataset, steps_per_trainset: int,\n                     steps_per_epoch: int, validset: tf.data.Dataset, max_duration: int,\n                     classifier_file_name: str):\n    assert steps_per_trainset >= steps_per_epoch\n    n_epochs = int(round(10.0 * steps_per_trainset / float(steps_per_epoch)))\n    print(f'Maximal duration of the BERT-based classifier training is {max_duration} seconds.')\n    callbacks = [\n        tf.keras.callbacks.EarlyStopping(patience=7, monitor='val_loss', mode='min',\n                                         restore_best_weights=False, verbose=True),\n        tf.keras.callbacks.ModelCheckpoint(monitor='val_loss', mode=\"min\",\n                                           save_weights_only=True, save_best_only=True,\n                                           filepath=classifier_file_name),\n        tfa.callbacks.TimeStopping(seconds=max_duration, verbose=True)\n    ]\n    history = nn.fit(\n        trainset,\n        steps_per_epoch=steps_per_epoch,\n        validation_data=validset,\n        epochs=n_epochs,\n        callbacks=callbacks\n    )\n    show_training_process(history, 'loss')\n    nn.load_weights(classifier_file_name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def predict_with_classifier(texts: Dict[str, List[Tuple[str, int]]],\n                            tokenizer: BertTokenizer, maxlen: int,\n                            classifier: tf.keras.Model, batch_size: int,\n                            max_dataset_size: int = 0) -> \\\n        Dict[str, Tuple[np.ndarray, np.ndarray]]:\n    languages = sorted(list(texts.keys()))\n    predictions_by_languages = dict()\n    if max_dataset_size > 0:\n        max_size_per_lang = max_dataset_size // len(languages)\n        err_msg = '{0} is too small number of dataset samples!'.format(max_dataset_size)\n        assert max_size_per_lang > 0, err_msg\n    else:\n        max_size_per_lang = 0\n    for cur_lang in languages:\n        selected_indices = list(range(len(texts[cur_lang])))\n        if max_size_per_lang > 0:\n            if len(selected_indices) > max_size_per_lang:\n                selected_indices = random.sample(\n                    population=selected_indices,\n                    k=max_size_per_lang\n                )\n        tokens_of_texts, mask_of_texts = regular_encode(\n            texts=[texts[cur_lang][idx][0] for idx in selected_indices],\n            tokenizer=tokenizer, maxlen=maxlen\n        )\n        predictions = []\n        n_batches = int(np.ceil(len(selected_indices) / float(batch_size)))\n        for batch_idx in range(n_batches):\n            batch_start = batch_idx * batch_size\n            batch_end = min(len(selected_indices), batch_start + batch_size)\n            res = classifier.predict_on_batch(\n                [\n                    tokens_of_texts[batch_start:batch_end],\n                    mask_of_texts[batch_start:batch_end]\n                ]\n            )\n            if not isinstance(res, np.ndarray):\n                res = res.numpy()\n            predictions.append(res.reshape((res.shape[0],)))\n            del res\n        predictions = np.concatenate(predictions)\n        identifiers = np.array(\n            [texts[cur_lang][idx][1] for idx in selected_indices],\n            dtype=np.int32\n        )\n        predictions_by_languages[cur_lang] = (predictions, identifiers)\n        del predictions, identifiers, selected_indices\n    return predictions_by_languages","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"def show_roc_auc(y_true: np.ndarray, probabilities: np.ndarray, label: str,\n                 figure_id: int=1):\n    plt.figure(figure_id, figsize=(5, 5))\n    plt.plot([0, 1], [0, 1], 'k--')\n    print('ROC-AUC score for {0} is {1:.9f}'.format(\n        label, roc_auc_score(y_true=y_true, y_score=probabilities)\n    ))\n    fpr, tpr, _ = roc_curve(y_true=y_true, y_score=probabilities)\n    plt.plot(fpr, tpr, label=label.title())\n    plt.xlabel('False positive rate')\n    plt.ylabel('True positive rate')\n    plt.title('ROC curve')\n    plt.legend(loc='best')\n    plt.show()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_start_time = time.time()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"try:\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\n    max_seq_len = 256\n    batch_size_for_bert = 8 * strategy.num_replicas_in_sync\nelse:\n    strategy = tf.distribute.get_strategy()\n    physical_devices = tf.config.list_physical_devices('GPU')\n    for device_idx in range(strategy.num_replicas_in_sync):\n        tf.config.experimental.set_memory_growth(physical_devices[device_idx], True)\n    max_seq_len = 256\n    batch_size_for_bert = 4 * strategy.num_replicas_in_sync\nmodel_name = 'bert-base-multilingual-cased'\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\nprint('Model name: {0}'.format(model_name))\nprint('Maximal length of sequence is {0}'.format(max_seq_len))\nprint('Batch size for the BERT is {0}'.format(batch_size_for_bert))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"random.seed(42)\nnp.random.seed(42)\ntf.random.set_seed(42)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"tfa.register_all()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_learning_rate = 1e-5\ndataset_dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification'\nfinal_classifier_name = '/kaggle/working/mbert_for_toxicity.h5'","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"bert_tokenizer = AutoTokenizer.from_pretrained(model_name)\nbert_config = BertConfig.from_pretrained(model_name)\nprint(bert_config)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"sentence_embedding_size = bert_config.hidden_size\nprint('Sentence embedding size is {0}'.format(sentence_embedding_size))\nassert max_seq_len <= bert_config.max_position_embeddings","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"corpus_for_training = load_train_set(\n    os.path.join(dataset_dir, \"jigsaw-toxic-comment-train.csv\"),\n    text_field=\"comment_text\", lang_field=\"lang\",\n    sentiment_fields=[\"toxic\", \"severe_toxic\", \"obscene\", \"threat\", \"insult\",\n                      \"identity_hate\"]\n)\nassert 'en' in corpus_for_training","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"multilingual_corpus = load_train_set(\n    os.path.join(dataset_dir, \"validation.csv\"),\n    text_field=\"comment_text\", lang_field=\"lang\", sentiment_fields=[\"toxic\", ]\n)\nassert 'en' not in multilingual_corpus\nmax_size = 0\nprint('Multilingual data:')\nfor language in sorted(list(multilingual_corpus.keys())):\n    print('  {0}\\t\\t{1} samples'.format(language, len(multilingual_corpus[language])))\n    assert set(map(lambda cur: cur[1], multilingual_corpus[language])) == {0, 1}\n    if len(multilingual_corpus[language]) > max_size:\n        max_size = len(multilingual_corpus[language])","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"nonenglish_languages = sorted(list(multilingual_corpus.keys()))\ncorpus_for_validation = dict()\nfor lang in nonenglish_languages:\n    random.shuffle(multilingual_corpus[lang])\n    n = len(multilingual_corpus[lang]) // 2\n    corpus_for_validation[lang] = multilingual_corpus[lang][0:n]\n    corpus_for_training[lang] = multilingual_corpus[lang][n:]\n    del multilingual_corpus[lang]","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"texts_for_submission = load_test_set(\n    os.path.join(dataset_dir, \"test.csv\"),\n    text_field=\"content\", lang_field=\"lang\", id_field=\"id\"\n)\nprint('Data for submission:')\nfor language in sorted(list(texts_for_submission.keys())):\n    print('  {0}\\t\\t{1} samples'.format(language, len(texts_for_submission[language])))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_for_training, n_batches_per_data = build_dataset(\n    texts=corpus_for_training, dataset_size=150000,\n    tokenizer=bert_tokenizer, maxlen=max_seq_len,\n    batch_size=batch_size_for_bert, shuffle=True\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"dataset_for_validation, n_batches_per_epoch = build_dataset(\n    texts=corpus_for_validation, dataset_size=6000,\n    tokenizer=bert_tokenizer, maxlen=max_seq_len,\n    batch_size=batch_size_for_bert, shuffle=False\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del corpus_for_training\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"preparing_duration = int(round(time.time() - experiment_start_time))\nprint(\"Duration of data loading and preparing to the Siamese NN training is \"\n      \"{0} seconds.\".format(preparing_duration))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with strategy.scope():\n    bert_based_classifier = build_classifier(\n        transformer_name=model_name,\n        hidden_state_size=sentence_embedding_size,\n        max_len=max_seq_len,\n        lr=bert_learning_rate\n    )","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train_classifier(nn=bert_based_classifier, trainset=dataset_for_training,\n                 steps_per_trainset=n_batches_per_data,\n                 steps_per_epoch=min(5 * n_batches_per_epoch, n_batches_per_data),\n                 validset=dataset_for_validation,\n                 max_duration=int(round(2.0 * 3600.0 - preparing_duration)),\n                 classifier_file_name=final_classifier_name)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"val_predictions = predict_with_classifier(\n    texts=corpus_for_validation,\n    tokenizer=bert_tokenizer, maxlen=max_seq_len,\n    classifier=bert_based_classifier,\n    batch_size=batch_size_for_bert\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del corpus_for_validation\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"calculated_probas = []\ntrue_labels = []\nfor lang in val_predictions:\n    probabilities_, true_labels_ = val_predictions[lang]\n    calculated_probas.append(probabilities_)\n    true_labels.append(true_labels_)\ncalculated_probas = np.concatenate(calculated_probas)\ntrue_labels = np.concatenate(true_labels)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del val_predictions\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"show_roc_auc(y_true=true_labels, probabilities=calculated_probas,\n             label='multi')","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del true_labels, calculated_probas","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"del dataset_for_training\ndel dataset_for_validation\ngc.collect()","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"experiment_duration = int(round(time.time() - experiment_start_time))\nprint('Duration of BERT training is {0} seconds.'.format(\n    experiment_duration))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"final_predictions = predict_with_classifier(\n    texts=texts_for_submission,\n    tokenizer=bert_tokenizer, maxlen=max_seq_len,\n    classifier=bert_based_classifier,\n    batch_size=batch_size_for_bert\n)","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"with codecs.open('submission.csv', mode='w', encoding='utf-8', errors='ignore') as fp:\n    fp.write('id,toxic\\n')\n    for lang in final_predictions:\n        probabilities, IDs = final_predictions[lang]\n        assert probabilities.shape == IDs.shape\n        assert len(probabilities.shape) == 1\n        for sample_idx in range(probabilities.shape[0]):\n            id_val = IDs[sample_idx]\n            proba_val = probabilities[sample_idx]\n            fp.write('{0},{1:.9f}\\n'.format(id_val, proba_val))","metadata":{"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"print('Experiment duration is {0:.3f}.'.format(time.time() - experiment_start_time))","metadata":{"trusted":true},"execution_count":null,"outputs":[]}]}