{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":true},"cell_type":"code","source":"import os\nimport gc\nimport re\nimport keras\nimport pickle\nimport string\nimport random\nimport warnings\nimport matplotlib\nimport numpy as np\nimport pandas as pd\nimport transformers\nimport seaborn as sns\nimport tensorflow as tf\nimport keras.backend as K\nimport matplotlib.pyplot as plt\n\nfrom tokenizers import *\nfrom transformers import *\nfrom sklearn.metrics import *\nfrom tqdm.notebook import tqdm\nfrom kaggle_datasets import KaggleDatasets\nfrom sklearn.model_selection import StratifiedKFold\n\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.metrics import AUC\nfrom tensorflow.keras.callbacks import *\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.callbacks import Callback\n\n\nwarnings.filterwarnings(\"ignore\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data_path = \"../input/jigsaw-multilingual-toxic-comment-classification/\"\ntest_en_path = \"../input/jigsaw-ml-laser-embed-without-cleaning-en/\"\nval_en_path = \"../input/val-en-df/\"\n\nlaser_path = \"../input/jigsaw-ml-laser-embed-without-cleaning/\"\nuse_path = \"../input/jigsaw-multilingual-use-embeddings/\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"PRETRAINED_TOKENIZER = 'jplu/tf-xlm-roberta-large'\nPRETRAINED_MODEL     = '/kaggle/input/jigsaw-ml-xlm-roberta-finetune'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def seed_everything(seed):\n    random.seed(seed)\n    np.random.seed(seed)\n    os.environ[\"PYTHONHASHSEED\"] = str(seed)\n    tf.random.set_seed(seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Setup","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\nAUTO = tf.data.experimental.AUTOTUNE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class Config:\n    seed = 42\n    \n    # Architecture\n    n_laser = 1024\n    n_use = 512\n    \n    laser_ft = 1024\n    use_ft = 512\n    logit_ft = 2048\n    \n    # Training\n    k = 4\n    \n    batch_size = 16 * strategy.num_replicas_in_sync\n    max_len = 192\n    epochs = 1\n    \n    lr = 8e-6\n    min_lr = 8e-6\n    warmup_prop = 0.1\n    weight_decay = 0.\n    \nconfig = Config()\nseed_everything(config.seed)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"def read_pickle_from_file(pickle_file):\n    with open(pickle_file,'rb') as f:\n        x = pickle.load(f)\n    return x\n\ndef write_pickle_to_file(pickle_file, x):\n    with open(pickle_file, 'wb') as f:\n        pickle.dump(x, f, pickle.HIGHEST_PROTOCOL)\n        \ndef read_embed(path, name, file_name):\n    em = read_pickle_from_file(path + file_name)\n\n    columns = ['{}_{}'.format(name, i) for i in range(em.shape[1])]\n    df = pd.DataFrame(em, columns=columns)\n    del em  \n    \n    return df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jigsaw_toxic_df = pd.read_csv(data_path + \"jigsaw-toxic-comment-train.csv\")\njigsaw_toxic_df['lang'] = 'en'\n\n# jigsaw_bias_df = pd.read_csv(data_path + \"jigsaw-unintended-bias-train.csv\")\n# jigsaw_bias_df['toxic'] = jigsaw_bias_df['toxic'].round().astype(int)\n# jigsaw_bias_df['lang'] = 'en'\n\n\nvalid_df = read_pickle_from_file(test_en_path + 'valid_en_df.pkl')\ntest_df = read_pickle_from_file(test_en_path + 'test_en_df.pkl')\nsub_df = pd.read_csv(data_path + 'sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"Jigsaw toxic : {len(jigsaw_toxic_df)} texts\")\n# print(f\"Jigsaw bias : {len(jigsaw_bias_df)} texts\")\nprint(f\"Validation : {len(valid_df)} texts\")\nprint(f\"Test : {len(sub_df)} texts\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sns.countplot(valid_df['lang'])\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Laser Embedding","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"laser_toxic = read_embed(laser_path, 'laser', 'train1_em.pkl')\n# laser_bias = read_embed(laser_path, 'laser', 'train2_em.pkl')\n\nlaser_test = read_embed(laser_path, 'laser', 'test_em.pkl')\nlaser_test_en = read_embed(test_en_path, 'laser', 'test_en_em.pkl')\n\nlaser_val = read_embed(laser_path, 'laser', 'valid_em.pkl')\nlaser_val_en = read_embed(test_en_path, 'laser', 'valid_en_em.pkl')\n\nlaser_columns = laser_toxic.columns\nn_lasers = len(laser_columns)\nprint(\"Laser embedding dimension :\", n_lasers)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## USE Embeddings","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"use_toxic = read_embed(use_path, 'use', 'train1_em.pkl')\n# use_bias = read_embed(use_path, 'use', 'train2_em.pkl')\n\nuse_test = read_embed(use_path, 'use', 'test_em.pkl')\nuse_test_en = read_embed(use_path, 'use', 'test_en_em.pkl')\n\nuse_val = read_embed(use_path, 'use', 'valid_em.pkl')\nuse_val_en = read_embed(use_path, 'use', 'valid_en_em.pkl')\n\nuse_columns = use_toxic.columns\nn_uses = len(use_columns)\nprint(\"Use embedding dimension :\", n_uses)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Remove Outliers\n- Senences that have a far-reaching meaning","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# train_df = jigsaw_toxic_df.copy()\n# train_df['check_embed'] = 0\n# count_old = len(train_df)\n# l2_dist = lambda x, y: np.power(x - y, 2).sum(axis=1)\n\n# for i in range(2):\n#     index0 = np.where((train_df['toxic'].values==0) & (train_df['check_embed'].values==0))\n#     index1 = np.where((train_df['toxic'].values==1) & (train_df['check_embed'].values==0))\n    \n#     toxic_ave_0 = np.average(use_toxic.values[index0], axis=0)\n#     toxic_ave_1 = np.average(use_toxic.values[index1], axis=0)\n    \n#     train_df['toxic_0_l2'] = np.array(l2_dist(toxic_ave_0, use_toxic.values))\n#     train_df['toxic_1_l2'] = np.array(l2_dist(toxic_ave_1, use_toxic.values))   \n    \n#     select_0 = train_df['toxic_0_l2'].values[index0].argsort()[::-1][:20]\n#     select_1 = train_df['toxic_1_l2'].values[index1].argsort()[::-1][:5]\n    \n#     select_0 = [index0[0][x] for x in select_0]\n#     select_1 = [index1[0][x] for x in select_1]\n    \n#     train_df.loc[select_0, 'check_embed'] = 1\n#     train_df.loc[select_1, 'check_embed'] = 1  \n    \n# train_df = train_df[train_df.check_embed==0].copy()\n# train_df.reset_index(drop=True, inplace=True)\n# train_df.drop(['check_embed', 'toxic_0_l2', 'toxic_1_l2'], axis=1, inplace=True)    \n\n# count_new = len(train_df)\n# print(f\"Removed {count_old - count_new} texts\")\n\n# jigsaw_toxic_df = train_df","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Optimizer","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"import tensorflow as tf\nfrom tensorflow.python.keras.optimizer_v2.optimizer_v2 import OptimizerV2\nfrom tensorflow.python import ops, math_ops, state_ops, control_flow_ops\nfrom tensorflow.python.keras import backend_config\n\n\nclass AdamWarmup(OptimizerV2):\n    \"\"\"Adam optimizer with warmup.\"\"\"\n\n    def __init__(self,\n                 decay_steps,\n                 warmup_steps,\n                 min_lr=0.0,\n                 learning_rate=0.001,\n                 beta_1=0.9,\n                 beta_2=0.999,\n                 epsilon=1e-7,\n                 weight_decay=0.,\n                 weight_decay_pattern=None,\n                 amsgrad=False,\n                 name='Adam',\n                 **kwargs):\n        r\"\"\"Construct a new Adam optimizer.\n        Args:\n            decay_steps: Learning rate will decay linearly to zero in decay steps.\n            warmup_steps: Learning rate will increase linearly to lr in first warmup steps.\n            lr: float >= 0. Learning rate.\n            beta_1: float, 0 < beta < 1. Generally close to 1.\n            beta_2: float, 0 < beta < 1. Generally close to 1.\n            epsilon: float >= 0. Fuzz factor. If `None`, defaults to `K.epsilon()`.\n            weight_decay: float >= 0. Weight decay.\n            weight_decay_pattern: A list of strings. The substring of weight names to be decayed.\n                                  All weights will be decayed if it is None.\n            amsgrad: boolean. Whether to apply the AMSGrad variant of this\n                algorithm from the paper \"On the Convergence of Adam and\n                Beyond\".\n        \"\"\"\n\n        super(AdamWarmup, self).__init__(name, **kwargs)\n        self._set_hyper('decay_steps', float(decay_steps))\n        self._set_hyper('warmup_steps', float(warmup_steps))\n        self._set_hyper('min_lr', min_lr)\n        self._set_hyper('learning_rate', kwargs.get('lr', learning_rate))\n        self._set_hyper('decay', self._initial_decay)\n        self._set_hyper('beta_1', beta_1)\n        self._set_hyper('beta_2', beta_2)\n        self._set_hyper('weight_decay', weight_decay)\n        self.epsilon = epsilon or backend_config.epsilon()\n        self.amsgrad = amsgrad\n        self._initial_weight_decay = weight_decay\n        self._weight_decay_pattern = weight_decay_pattern\n        \n        self.current_lr = self.lr\n\n    def _create_slots(self, var_list):\n        for var in var_list:\n            self.add_slot(var, 'm')\n        for var in var_list:\n            self.add_slot(var, 'v')\n        if self.amsgrad:\n            for var in var_list:\n                self.add_slot(var, 'vhat')\n\n    def set_weights(self, weights):\n        params = self.weights\n        num_vars = int((len(params) - 1) / 2)\n        if len(weights) == 3 * num_vars + 1:\n            weights = weights[:len(params)]\n        super(AdamWarmup, self).set_weights(weights)\n\n    def _resource_apply_dense(self, grad, var):\n        var_dtype = var.dtype.base_dtype\n        lr_t = self._decayed_lr(var_dtype)\n        m = self.get_slot(var, 'm')\n        v = self.get_slot(var, 'v')\n        beta_1_t = self._get_hyper('beta_1', var_dtype)\n        beta_2_t = self._get_hyper('beta_2', var_dtype)\n        epsilon_t = ops.convert_to_tensor(self.epsilon, var_dtype)\n        local_step = math_ops.cast(self.iterations + 1, var_dtype)\n        beta_1_power = math_ops.pow(beta_1_t, local_step)\n        beta_2_power = math_ops.pow(beta_2_t, local_step)\n\n        decay_steps = self._get_hyper('decay_steps', var_dtype)\n        warmup_steps = self._get_hyper('warmup_steps', var_dtype)\n        min_lr = self._get_hyper('min_lr', var_dtype)\n        lr_t = tf.where(\n            local_step <= warmup_steps,\n            lr_t * (local_step / warmup_steps),\n            min_lr + (lr_t - min_lr) * (1.0 - tf.minimum(local_step, decay_steps) / decay_steps),\n        )\n        lr_t = (lr_t * math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power))\n\n        m_t = state_ops.assign(m,\n                               beta_1_t * m + (1.0 - beta_1_t) * grad,\n                               use_locking=self._use_locking)\n\n        v_t = state_ops.assign(v,\n                               beta_2_t * v + (1.0 - beta_2_t) * math_ops.square(grad),\n                               use_locking=self._use_locking)\n\n        if self.amsgrad:\n            v_hat = self.get_slot(var, 'vhat')\n            v_hat_t = math_ops.maximum(v_hat, v_t)\n            var_update = m_t / (math_ops.sqrt(v_hat_t) + epsilon_t)\n        else:\n            var_update = m_t / (math_ops.sqrt(v_t) + epsilon_t)\n\n        if self._initial_weight_decay > 0.0:\n            weight_decay = self._get_hyper('weight_decay', var_dtype)\n            var_update += weight_decay * var\n        var_update = state_ops.assign_sub(var, lr_t * var_update, use_locking=self._use_locking)\n\n        updates = [var_update, m_t, v_t]\n        if self.amsgrad:\n            updates.append(v_hat_t)\n        return control_flow_ops.group(*updates)\n\n    def _resource_apply_sparse(self, grad, var, indices):\n        var_dtype = var.dtype.base_dtype\n        lr_t = self._decayed_lr(var_dtype)\n        beta_1_t = self._get_hyper('beta_1', var_dtype)\n        beta_2_t = self._get_hyper('beta_2', var_dtype)\n        epsilon_t = ops.convert_to_tensor(self.epsilon, var_dtype)\n        local_step = math_ops.cast(self.iterations + 1, var_dtype)\n        beta_1_power = math_ops.pow(beta_1_t, local_step)\n        beta_2_power = math_ops.pow(beta_2_t, local_step)\n\n        decay_steps = self._get_hyper('decay_steps', var_dtype)\n        warmup_steps = self._get_hyper('warmup_steps', var_dtype)\n        min_lr = self._get_hyper('min_lr', var_dtype)\n        lr_t = tf.where(\n            local_step <= warmup_steps,\n            lr_t * (local_step / warmup_steps),\n            min_lr + (lr_t - min_lr) * (1.0 - tf.minimum(local_step, decay_steps) / decay_steps),\n        )\n        lr_t = (lr_t * math_ops.sqrt(1 - beta_2_power) / (1 - beta_1_power))\n\n        m = self.get_slot(var, 'm')\n        m_scaled_g_values = grad * (1 - beta_1_t)\n        m_t = state_ops.assign(m, m * beta_1_t, use_locking=self._use_locking)\n        with ops.control_dependencies([m_t]):\n            m_t = self._resource_scatter_add(m, indices, m_scaled_g_values)\n\n        v = self.get_slot(var, 'v')\n        v_scaled_g_values = (grad * grad) * (1 - beta_2_t)\n        v_t = state_ops.assign(v, v * beta_2_t, use_locking=self._use_locking)\n        with ops.control_dependencies([v_t]):\n            v_t = self._resource_scatter_add(v, indices, v_scaled_g_values)\n\n        if self.amsgrad:\n            v_hat = self.get_slot(var, 'vhat')\n            v_hat_t = math_ops.maximum(v_hat, v_t)\n            var_update = m_t / (math_ops.sqrt(v_hat_t) + epsilon_t)\n        else:\n            var_update = m_t / (math_ops.sqrt(v_t) + epsilon_t)\n\n        if self._initial_weight_decay > 0.0:\n            weight_decay = self._get_hyper('weight_decay', var_dtype)\n            var_update += weight_decay * var\n        var_update = state_ops.assign_sub(var, lr_t * var_update, use_locking=self._use_locking)\n\n        updates = [var_update, m_t, v_t]\n        if self.amsgrad:\n            updates.append(v_hat_t)\n        return control_flow_ops.group(*updates)\n\n    def get_config(self):\n        config = super(AdamWarmup, self).get_config()\n        config.update({\n            'decay_steps': self._serialize_hyperparameter('decay_steps'),\n            'warmup_steps': self._serialize_hyperparameter('warmup_steps'),\n            'min_lr': self._serialize_hyperparameter('min_lr'),\n            'learning_rate': self._serialize_hyperparameter('learning_rate'),\n            'decay': self._serialize_hyperparameter('decay'),\n            'beta_1': self._serialize_hyperparameter('beta_1'),\n            'beta_2': self._serialize_hyperparameter('beta_2'),\n            'weight_decay': self._serialize_hyperparameter('weight_decay'),\n            'epsilon': self.epsilon,\n            'amsgrad': self.amsgrad,\n        })\n        return config\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def mixed_loss(y_true, y_pred, beta=0.10):\n    loss = beta*focal_loss(y_true,y_pred) + (1-beta)*tf.keras.losses.binary_crossentropy(y_true, y_pred)\n    return loss","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow.keras.layers as KL\n\n\ndef nn_block(input_layer, size, dropout_rate, activation):\n    out_layer = KL.Dense(size, activation=None)(input_layer)\n    #out_layer = KL.BatchNormalization()(out_layer)\n    out_layer = KL.Activation(activation)(out_layer)\n    out_layer = KL.Dropout(dropout_rate)(out_layer)\n    return out_layer\n\ndef cnn_block(input_layer, size, dropout_rate, activation):\n    out_layer = KL.Conv1D(size, 1, activation=None)(input_layer)\n    #out_layer = KL.LayerNormalization()(out_layer)\n    out_layer = KL.Activation(activation)(out_layer)\n    out_layer = KL.Dropout(dropout_rate)(out_layer)\n    return out_layer\n\ndef build_model(transformer, config):\n    # transformer\n    input_ids = Input(shape=(config.max_len,), dtype=tf.int64, name=\"input_ids\")\n    input_masks = Input(shape=(config.max_len,), dtype=tf.int64, name=\"input_masks\")\n    input_segments = Input(shape=(config.max_len,), dtype=tf.int64, name=\"input_segments\")\n    \n    sequence_output = transformer(input_ids, attention_mask=input_masks, token_type_ids=input_segments)[0]\n    \n    ave_pool = GlobalAveragePooling1D()(sequence_output)\n    max_pool = GlobalMaxPooling1D()(sequence_output)\n    \n    # lasers\n    lasers = Input(shape=(config.n_laser,), dtype=tf.float32, name=\"lasers\") \n    lasers_output = nn_block(lasers,config.laser_ft,0.1,'tanh')\n    #lasers_output = Dense(config.laser_ft, activation='tanh')(lasers)\n    \n     # uses\n    uses = Input(shape=(config.n_use,), dtype=tf.float32, name=\"uses\") \n    uses_output = nn_block(uses,config.use_ft,0.1,'tanh')\n\n  #  uses_output = Dense(config.use_ft, activation='tanh')(uses)   \n    \n    features = Concatenate()([ave_pool, max_pool, KL.BatchNormalization()(lasers_output), KL.BatchNormalization()(uses_output)])\n    \n    outs = []\n    for _ in range(5):\n        x = Dropout(0.5)(features)\n        x = tf.keras.layers.Dense(config.logit_ft, activation='tanh')(x)\n        x = Dense(1, activation='sigmoid')(x)\n        outs.append(x)\n    \n    out = Average()(outs)\n    \n    model = Model(inputs=[input_ids, input_masks, input_segments, lasers, uses], outputs=out)\n#     model.compile(optimizer, loss=loss, metrics=[AUC()])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = build_model(TFRobertaModel.from_pretrained(PRETRAINED_MODEL), config)\n\nmodel.summary()\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenizer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=True, \n        return_token_type_ids=True,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return [np.asarray(enc_di['input_ids'], dtype=np.int64), \n            np.asarray(enc_di['attention_mask'], dtype=np.int64), \n            np.asarray(enc_di['token_type_ids'], dtype=np.int64)]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepare_dataset(x, laser, use, y=None, mode=\"train\", batch_size=16):\n    if y is None:\n        y = np.zeros(len(x[0]))\n        \n    dataset = tf.data.Dataset.from_tensor_slices((\n        {\n            \"input_ids\": x[0], \n            \"input_masks\": x[1],\n            \"input_segments\": x[2], \n            \"lasers\": laser,\n            \"uses\": use\n        }, \n        y\n    ))\n    if mode == \"train\":\n        dataset = dataset.repeat().shuffle(2048).batch(batch_size).prefetch(AUTO)\n    elif mode == \"val\":\n        dataset = dataset.batch(batch_size)#.cache().prefetch(AUTO)\n    else: #test\n        dataset = dataset.batch(batch_size)\n        \n    return dataset","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"COLUMNS = ['id', 'comment_text', 'toxic', 'lang']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = AutoTokenizer.from_pretrained(PRETRAINED_TOKENIZER)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Test data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = regular_encode(test_df['content'].values, tokenizer, maxlen=config.max_len)\nx_en_test = regular_encode(test_df['content_en'].values, tokenizer, maxlen=config.max_len)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = prepare_dataset(x_test, laser_test.values, use_test.values, batch_size=config.batch_size, mode='test')\ntest_en_dataset = prepare_dataset(x_en_test, laser_test_en.values, use_test_en.values, batch_size=config.batch_size, mode='test')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### $k$-fold","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"WEIGHTS = [(1, 0), (0.9, 0.1),(0.8, 0.2), (0.7, 0.3), (0.6, 0.4), (0.5, 0.5)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"SELECTED_FOLDS = [2,3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = build_model(TFRobertaModel.from_pretrained(PRETRAINED_MODEL), config)\nmodel.save_weights('model_checkpoint.h5')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_history_list = []\n\ntest_preds  = np.zeros(len(test_df))\ntest_preds_en  = np.zeros(len(test_df))\npred_oof = np.zeros(len(valid_df))\npred_oof_en = np.zeros(len(valid_df))\n\nsplits = list(StratifiedKFold(n_splits=config.k, shuffle=True, random_state=config.seed).split(valid_df['toxic'].values, valid_df['toxic'].values))\n\nfor k, (train_idx, val_idx) in enumerate(splits):\n    \n    if k not in SELECTED_FOLDS:\n        continue\n        \n    seed_everything(config.seed + k)\n    print(f'\\n\\t -> Fold {k+1}\\n')\n    \n    print(f' - Data Preparation \\n')\n    \n    df_train = pd.concat([jigsaw_toxic_df[COLUMNS], valid_df.iloc[train_idx]])\n    use_train =  pd.concat([use_toxic, use_val.iloc[train_idx]])\n    laser_train =  pd.concat([laser_toxic, laser_val.iloc[train_idx]])\n    \n    x_train = regular_encode(df_train['comment_text'].values, tokenizer, maxlen=config.max_len)\n    y_train = df_train['toxic'].values\n    \n    df_val = valid_df.iloc[val_idx]\n    use_val_ = use_val.iloc[val_idx]\n    use_val_en_ = use_val_en.iloc[val_idx]\n    laser_val_ = laser_val.iloc[val_idx]\n    laser_val_en_ = laser_val_en.iloc[val_idx]\n    \n    x_val = regular_encode(df_val['comment_text'].values, tokenizer, maxlen=config.max_len)\n    x_val_en = regular_encode(df_val['comment_text_en'].values, tokenizer, maxlen=config.max_len)\n    y_val = df_val['toxic'].values\n\n    train_dataset = prepare_dataset(x_train, laser_train, use_train, y=y_train, batch_size=config.batch_size, mode='train')\n    val_dataset = prepare_dataset(x_val, laser_val_, use_val_, y=y_val, batch_size=config.batch_size, mode='val')\n    val_dataset_en = prepare_dataset(x_val_en, laser_val_en_, use_val_en_, y=y_val, batch_size=config.batch_size, mode='val')\n\n    print(' - Model Preparation \\n')\n    \n    steps_per_epoch = len(x_train[0]) // config.batch_size\n    steps = steps_per_epoch * config.epochs\n    \n    optimizer = AdamWarmup(\n        lr=config.lr, \n        min_lr=config.min_lr,\n        decay_steps=steps, \n        warmup_steps=int(steps * config.warmup_prop),\n        weight_decay=config.weight_decay\n    )\n    \n    with strategy.scope():\n        model.compile(optimizer, loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC()])\n        model.load_weights('model_checkpoint.h5')\n\n    print(f' - Training \\n')\n\n    train_history = model.fit(\n        train_dataset,\n        steps_per_epoch=steps_per_epoch,\n        validation_data=val_dataset,\n        epochs=config.epochs\n    )\n    \n    train_history_list.append(train_history)\n#     model.save_weights(f'checkpoint_{k+1}.h5')\n    \n    print(f'\\n - Predicting \\n')\n    \n    pred_val = model.predict(val_dataset, verbose=0).reshape(-1)\n    pred_oof[val_idx] = pred_val\n    \n    pred_val_en = model.predict(val_dataset_en, verbose=0).reshape(-1)\n    pred_oof_en[val_idx] = pred_val_en \n    \n    for weights in WEIGHTS:\n        pred = pred_val * weights[0] + pred_val_en * weights[1]\n        score = roc_auc_score(y_val, pred)\n        print(f'Scored {score:.4f} with weights {weights}\\n')\n    \n    test_preds += model.predict(test_dataset, verbose=1).reshape(-1) / 2.0\n    test_preds_en += model.predict(test_en_dataset, verbose=1).reshape(-1) / 2.0\n    \n    del train_dataset, val_dataset, val_dataset_en, x_train, x_val, x_val_en, \n    del use_val_, use_val_en_, laser_val_, laser_val_en_, use_train, laser_train, df_train, df_val\n    gc.collect()\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n#     break","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for weights in WEIGHTS:\n    pred = pred_oof * weights[0] + pred_oof_en * weights[1]        \n    score = roc_auc_score(valid_df['toxic'].values, pred)\n\n    print(f' -> Local CV score is {score:.4f} for weights {weights} \\n')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def history_to_dataframe( history ):\n    columns = list(history.history.keys())\n    datas   = list(history.history.values())\n    return pd.DataFrame(np.array(datas).T, columns=columns)\n   \nfor k, history in enumerate(train_history_list):\n    df = history_to_dataframe( history )\n    print('*' * 20)\n    print('K:', k+1)\n    print(df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save(\"pred_test.npy\", test_preds)\nnp.save(\"pred_test_en.npy\", test_preds_en)\nnp.save(\"pred_oof_en.npy\", pred_oof_en)\nnp.save(\"pred_oof.npy\", pred_oof)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"for i, weights in enumerate(WEIGHTS):\n    preds = test_preds * weights[0] + test_preds_en * weights[1]      \n    \n    sub_df['toxic'] = preds\n    sub_df.to_csv(f'submission_{i}_{config.seed}.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"!ls","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}