{"cells":[{"metadata":{},"cell_type":"markdown","source":"### References\nRun a couple of experiments and ensembles based on the following notebooks and ideas\n* Notebook 1: [Link](https://www.kaggle.com/xhlulu/jigsaw-tpu-xlm-roberta)\n* Notebook 2: [Link](https://www.kaggle.com/swannnn/jigsaw-tpu-xlm-roberta-e3ad07)\n* Discussion 1: [Link](https://www.kaggle.com/c/jigsaw-multilingual-toxic-comment-classification/discussion/160980)"},{"metadata":{},"cell_type":"markdown","source":"## Import libraries"},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\nimport os\nimport gc\nimport numpy as np\nimport pandas as pd\nimport re\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\nfrom tensorflow.keras.layers import Dense, Dropout, Input, Concatenate, Average, GlobalAveragePooling1D, GlobalMaxPooling1D\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model, load_model\nfrom tensorflow.keras.callbacks import ReduceLROnPlateau, LearningRateScheduler, EarlyStopping, ModelCheckpoint\nfrom kaggle_datasets import KaggleDatasets\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tqdm.notebook import tqdm\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Helper Functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"def clean_text(text):\n    text = str(text)\n    text = re.sub(r'[0-9\"]', '', text) # number\n    text = re.sub(r'#[\\S]+\\b', '', text) # hash\n    text = re.sub(r'@[\\S]+\\b', '', text) # mention\n    text = re.sub(r'https?\\S+', '', text) # link\n    text = re.sub(r'\\s+', ' ', text) # multiple white spaces\n#     text = re.sub(r'\\W+', ' ', text) # non-alphanumeric\n    return text.strip()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def text_process(text):\n    ws = text.split(' ')\n    if(len(ws)>160):\n        text = ' '.join(ws[:160]) + ' ' + ' '.join(ws[-32:])\n    return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def dict_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_mask=True, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen,\n        truncation=True\n    )\n    \n    return {\n        \"input_ids\": np.array(enc_di['input_ids']),\n        \"attention_mask\": np.array(enc_di['attention_mask'])\n    }","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# First load the real tokenizer\ntokenizer = AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU configs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Configuration\nMODEL = 'jplu/tf-xlm-roberta-large'\nAUTO = tf.data.experimental.AUTOTUNE\nSEED = 2020\nEPOCHS_1 = 20\nEPOCHS_2 = 2\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nMAX_LEN = 192\nSHUFFLE = 2048\nVERBOSE = 1","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Read data"},{"metadata":{"trusted":true},"cell_type":"code","source":"train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\ntrain2 = pd.concat([train2[['comment_text', 'toxic']].query('toxic<=0.4'),\n                    train2[['comment_text', 'toxic']].query('toxic>=0.6')])\ntrain2.toxic = train2.toxic.round().astype(int)\ntrain2 = train2.sample(frac = 1, random_state=SEED)\ntrain3 = pd.read_csv(\"/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv\")\ntrain4 = pd.read_csv(\"/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv\")\ntrain5 = pd.read_csv(\"/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv\")\ntrain6 = pd.read_csv(\"/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv\")\ntrain7 = pd.read_csv(\"/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv\")\ntrain8 = pd.read_csv(\"/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv\")\n\nvalid1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\nvalid2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv')\nvalid2.rename(columns={'comment_text': 'old_comment_text', 'translated': 'comment_text'}, inplace=True)\n\ntest = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/test.csv')\ntest1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\ntest2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')\n\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# sampling 50/50\ntrain = pd.concat([\n    train1[['comment_text', 'toxic']].query('toxic==1'),\n    train1[['comment_text', 'toxic']].query('toxic==0').sample(n=len(train1[train1.toxic == 1]), random_state=SEED),\n    train2[['comment_text', 'toxic']].query('toxic==1'),\n    train2[['comment_text', 'toxic']].query('toxic==0').sample(n=len(train2[train2.toxic == 1]), random_state=SEED),\n    train3[['comment_text', 'toxic']].query('toxic==1'),\n    train3[['comment_text', 'toxic']].query('toxic==0').sample(n=len(train3[train3.toxic == 1]), random_state=SEED),\n    train4[['comment_text', 'toxic']].query('toxic==1'),\n    train4[['comment_text', 'toxic']].query('toxic==0').sample(n=len(train4[train4.toxic == 1]), random_state=SEED),\n    train5[['comment_text', 'toxic']].query('toxic==1'),\n    train5[['comment_text', 'toxic']].query('toxic==0').sample(n=len(train5[train5.toxic == 1]), random_state=SEED),\n    train6[['comment_text', 'toxic']].query('toxic==1'),\n    train6[['comment_text', 'toxic']].query('toxic==0').sample(n=len(train6[train6.toxic == 1]), random_state=SEED),\n    train7[['comment_text', 'toxic']].query('toxic==1'),\n    train7[['comment_text', 'toxic']].query('toxic==0').sample(n=len(train7[train7.toxic == 1]), random_state=SEED),\n    train8[['comment_text', 'toxic']].query('toxic==1'),\n    train8[['comment_text', 'toxic']].query('toxic==0').sample(n=len(train8[train8.toxic == 1]), random_state=SEED),\n    \n])\n\ntrain.toxic = train.toxic.round().astype(int)\ntrain = train.sample(frac = 1, random_state=SEED)\n\nvalid = pd.concat([valid1, valid2])\nvalid = valid.sample(frac = 1, random_state=SEED)\n\n\ntrain['comment_text'] = train.apply(lambda x: clean_text(x['comment_text']), axis=1)\nvalid['comment_text'] = valid.apply(lambda x: clean_text(x['comment_text']), axis=1)\ntest1['content'] = test1.apply(lambda x: clean_text(x['content']), axis=1)\ntest2['translated'] = test2.apply(lambda x: clean_text(x['translated']), axis=1)\n\ntrain['comment_text'] = train['comment_text'].apply(lambda x: text_process(x))\nvalid['comment_text'] = valid['comment_text'].apply(lambda x: text_process(x))\ntest1['content'] = test1['content'].apply(lambda x: text_process(x))\ntest2['translated'] = test2['translated'].apply(lambda x: text_process(x))\n\n\nn_train_steps = train.shape[0] // (BATCH_SIZE*8)\nn_valid_steps = valid.shape[0] // (BATCH_SIZE)\n\ndel train1, train2, train3, train4, train5, train6, train7, train8\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time \nx_train = dict_encode(train.comment_text.values, tokenizer, maxlen=MAX_LEN)\nx_valid = dict_encode(valid.comment_text.values, tokenizer, maxlen=MAX_LEN)\n\nx_test1 = dict_encode(test1.content.values, tokenizer, maxlen=MAX_LEN)\nx_test2 = dict_encode(test2.translated.values, tokenizer, maxlen=MAX_LEN)\n\ny_train = train.toxic.values\ny_valid = valid.toxic.values\n\ndel train, valid\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Dataset objects"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(SHUFFLE)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\n\ntest1_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test1)\n    .batch(BATCH_SIZE)\n)\n\ntest2_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test2)\n    .batch(BATCH_SIZE)\n)\n\ndel x_train, x_valid, y_train, y_valid, x_test1, x_test2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Callbacks"},{"metadata":{"trusted":true},"cell_type":"code","source":"lrs = ReduceLROnPlateau(monitor='val_accuracy', mode ='max', factor = 0.7, min_lr= 1e-7, verbose = 1, patience = 2)\nes1 = EarlyStopping(monitor='val_accuracy', mode='max', verbose = 1, patience = 5, restore_best_weights=True)\nes2 = EarlyStopping(monitor='accuracy', mode='max', verbose = 1, patience = 1, restore_best_weights=True)\ncallbacks_list1 = [lrs,es1]\ncallbacks_list2 = [lrs,es2]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Build model"},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_ids\")\n    attention_mask = Input(shape=(max_len,), dtype=tf.int32, name=\"attention_mask\")\n    sequence_output = transformer({\"input_ids\": input_word_ids, \"attention_mask\": attention_mask})[0]\n\n    avg_pool = GlobalAveragePooling1D()(sequence_output)\n    max_pool = GlobalMaxPooling1D()(sequence_output)\n    cls_token = Concatenate()([avg_pool, max_pool])\n    \n    samples = []\n    for n in range(2):\n        sample = Dropout(0.4)(cls_token)\n        sample = Dense(1, activation='sigmoid', name=f'sample_{n}')(sample)\n        samples.append(sample)\n    \n    out = Average(name='output')(samples)\n    model = Model(inputs={\n                \"input_ids\": input_word_ids,\n                \"attention_mask\": attention_mask\n                }, \n                outputs=out,\n                name='XLM-R')\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=[tf.keras.metrics.AUC(name='auc'), 'accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Load to TPU"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Run model"},{"metadata":{"trusted":true},"cell_type":"code","source":"model_history_1 = model.fit(\n    train_dataset,\n    steps_per_epoch=n_train_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS_1,\n    callbacks=callbacks_list1,\n    verbose=VERBOSE\n )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_valid_steps,\n    epochs=EPOCHS_2,\n    callbacks=callbacks_list2,\n    verbose=VERBOSE\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Blending"},{"metadata":{"trusted":true},"cell_type":"code","source":"#Blending\nmulti_sub = model.predict(test1_dataset, verbose=1)\neng_sub = model.predict(test2_dataset, verbose=1)\n\nsub['toxic'] = multi_sub*0.75 + eng_sub*0.25\n\n#Post-processing\nsub.loc[test[\"lang\"] == \"es\", \"toxic\"] *= 1.11\nsub.loc[test[\"lang\"] == \"fr\", \"toxic\"] *= 1.06\nsub.loc[test[\"lang\"] == \"it\", \"toxic\"] *= 0.93\nsub.loc[test[\"lang\"] == \"pt\", \"toxic\"] *= 0.92\nsub.loc[test[\"lang\"] == \"tr\", \"toxic\"] *= 0.94\n\n# min-max normalize\nsub.toxic -= sub.toxic.min()\nsub.toxic /= (sub.toxic.max() - sub.toxic.min())\nsub.toxic.hist(bins=100, log=False, alpha=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Submission"},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}