{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"!pip install -q googletrans\nimport matplotlib.pyplot as plt\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport os\nimport re\n\nimport transformers\nimport tensorflow as tf\nfrom tqdm.notebook import tqdm\nfrom wordcloud import WordCloud, STOPWORDS\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tokenizers import BertWordPieceTokenizer\nfrom kaggle_datasets import KaggleDatasets\nfrom googletrans import Translator\nfrom colorama import Fore, Back, Style, init\nimport plotly.graph_objects as go\ntranslator = Translator()\n\nfrom tensorflow.keras.layers import (Dense, Input, LSTM, Bidirectional, Activation, Conv1D, \n                                     GRU,Embedding, Flatten, Dropout, Add, concatenate, MaxPooling1D,\n                                     GlobalAveragePooling1D,  GlobalMaxPooling1D, \n                                     GlobalMaxPool1D,SpatialDropout1D)\n\nfrom tensorflow.keras import (initializers, regularizers, constraints, \n                              optimizers, layers, callbacks)\n\nsns.set(style=\"darkgrid\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification'\n\ntrain_set1 = pd.read_csv(os.path.join(dir, 'jigsaw-toxic-comment-train.csv'))\ntrain_set2 = pd.read_csv(os.path.join(dir, 'jigsaw-unintended-bias-train.csv'))\ntrain_set2.toxic = train_set2.toxic.round().astype(int)\n\nvalid = pd.read_csv(os.path.join(dir, 'validation.csv'))\ntest = pd.read_csv(os.path.join(dir, 'test.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Combine train1 with a subset of train2\ntrain = pd.concat([\n    train_set1[['comment_text', 'toxic']],\n    train_set2[['comment_text', 'toxic']].query('toxic==1'),\n    train_set2[['comment_text', 'toxic']].query('toxic==0').sample(n=100000, random_state=0)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid.shape)\nvalid.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.toxic.value_counts())\nsns.countplot(train.toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid.toxic.value_counts())\nsns.countplot(valid.toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid.lang.value_counts())\nsns.countplot(valid.lang)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.lang.value_counts())\nsns.countplot(test.lang)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ax(rows=1, cols=2, size=7):\n    \"\"\"Return a Matplotlib Axes array to be used in\n    all visualizations in the notebook. Provide a\n    central point to control graph sizes.\n    \n    Adjust the size attribute to control how big to render images\n    \"\"\"\n    fig, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n    return fig, ax","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = get_ax()\n\nsns.distplot(train[train['toxic']==0]['comment_text'].str.len(), axlabel=\"Non Toxic\", ax=ax[0])\nsns.distplot(train[train['toxic']==0]['comment_text'].str.split().str.len(), axlabel=\"Non Toxic\", ax=ax[1])\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = get_ax()\n\nsns.distplot(train[train['toxic']==1]['comment_text'].str.len(), axlabel=\"Toxic\", ax=ax[0])\nsns.distplot(train[train['toxic']==1]['comment_text'].str.split().str.len(), axlabel=\"Toxic\", ax=ax[1])\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = set(STOPWORDS)\n\ndef show_wordcloud(data, title = None):\n    wordcloud = WordCloud(\n        background_color='white',\n        stopwords=stopwords,\n        max_words=50,\n        max_font_size=40, \n        scale=5,\n        random_state=1\n    ).generate(str(data))\n\n    fig = plt.figure(1, figsize=(10,10))\n    plt.axis('off')\n    if title: \n        fig.suptitle(title, fontsize=20)\n        fig.subplots_adjust(top=2.3)\n\n    plt.imshow(wordcloud)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_wordcloud(train['comment_text'].sample(20000), \n               title = '[Comment_Text] Prevalent Words')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_wordcloud(valid['comment_text'].sample(1000), \n               title = '[Comment_Text] Prevalent Words')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_wordcloud(test['content'].sample(1000), \n               title = '[Content] Prevalent Words')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(5):\n    print(f'[CONTENT {i}]\\n', train['comment_text'][i])\n    print()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fast encoder\ndef fast_encode(texts, tokenizer, chunk_size=240, maxlen=512):\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in range(0, len(texts), chunk_size):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)\n# general encoder\ndef regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Create strategy from tpu\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\n# Configuration\nEPOCHS = 4\nBATCH_SIZE = 16* strategy.num_replicas_in_sync\nMODEL = 'jplu/tf-xlm-roberta-large'\nMAX_LEN = 224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://huggingface.co/models\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\n\n#First load the real tokenizer\ntokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nx_train = regular_encode(train.comment_text.values, \n                         tokenizer, maxlen=MAX_LEN)\nx_valid = regular_encode(valid.comment_text.values, \n                         tokenizer, maxlen=MAX_LEN)\nx_test = regular_encode(test.content.values, tokenizer, \n                        maxlen=MAX_LEN)\n\ny_train = train.toxic.values\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\ndef label_smoothing(y_true,y_pred):\n     return tf.keras.losses.binary_crossentropy(y_true,y_pred,label_smoothing=0.15)\n\ndef focal_loss(gamma=2., alpha=.2):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    \n    cls_token = sequence_output[:, 0, :]\n    x = Dropout(0.3)(cls_token)\n    out = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss=label_smoothing,\n                  metrics=[tf.keras.metrics.AUC()]) # competition metrics\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow.keras.backend as K\n\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer, max_len=MAX_LEN)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import SVG\nSVG(tf.keras.utils.model_to_dot(model, dpi=70).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def callback():\n    cb = []\n    \"\"\"\n    Model-Checkpoint\n    \"\"\"\n    checkpoint = callbacks.ModelCheckpoint('model.h5',\n                                       save_best_only=True, \n                                       mode='min',\n                                       monitor='val_loss', #  \n                                       save_weights_only=True, verbose=0)\n\n    cb.append(checkpoint)\n    \n    # Callback that streams epoch results to a csv file.\n    log = callbacks.CSVLogger('log.csv')\n    cb.append(log)\n\n    return cb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"calls = callback()\nn_steps = x_train.shape[0] // BATCH_SIZE\n\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    callbacks = calls,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# repeat the training on validaton set \n# the intuition behind such operation is to tweak the model parameter for non-english \nn_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_model_preds(model, indices=[0, 17, 1, 24]):\n    comments = valid.comment_text.loc[indices].values.tolist()\n    preds = model.predict(x_valid[indices].reshape(len(indices), -1))\n\n    for idx, i in enumerate(indices):\n        if y_valid[i] == 0:\n            label = \"Non-toxic\"\n            color = f'{Fore.GREEN}'\n            symbol = '\\u2714'\n        else:\n            label = \"Toxic\"\n            color = f'{Fore.RED}'\n            symbol = '\\u2716'\n\n        print('{}{} {}'.format(color, str(idx+1) + \". \" + label, symbol))\n        print(f'{Style.RESET_ALL}')\n        print(\"ORIGINAL\")\n        print(comments[idx]); print(\"\")\n        print(\"TRANSLATED\")\n        print(translator.translate(comments[idx]).text)\n        fig = go.Figure()\n        if list.index(sorted(preds[:, 0]), preds[idx][0]) > 1:\n            yl = [preds[idx][0], 1 - preds[idx][0]]\n        else:\n            yl = [1 - preds[idx][0], preds[idx][0]]\n        fig.add_trace(go.Bar(x=['Non-Toxic', 'Toxic'], y=yl, marker=dict(color=[\"seagreen\", \"indianred\"])))\n        fig.update_traces(name=comments[idx])\n        fig.update_layout(xaxis_title=\"Labels\", yaxis_title=\"Probability\", template=\"plotly_white\", title_text=\"Predictions for validation comment #{}\".format(idx+1))\n        fig.show()\n        \nvisualize_model_preds(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(dir + '/sample_submission.csv')\nsub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}