{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"!pip install -q googletrans\nimport matplotlib.pyplot as plt\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport seaborn as sns\nimport os\nimport re\n\nimport transformers\nimport tensorflow as tf\nfrom tqdm.notebook import tqdm\nfrom wordcloud import WordCloud, STOPWORDS\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras import layers\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tokenizers import BertWordPieceTokenizer\nfrom kaggle_datasets import KaggleDatasets\nfrom googletrans import Translator\nfrom colorama import Fore, Back, Style, init\nimport plotly.graph_objects as go\ntranslator = Translator()\n\nfrom tensorflow.keras.layers import (Dense, Input, LSTM, Bidirectional, Activation, Conv1D, \n                                     GRU,Embedding, Flatten, Dropout, Add, concatenate, MaxPooling1D,\n                                     GlobalAveragePooling1D,  GlobalMaxPooling1D, \n                                     GlobalMaxPool1D,SpatialDropout1D)\n\nfrom tensorflow.keras import (initializers, regularizers, constraints, \n                              optimizers, layers, callbacks)\n\nsns.set(style=\"darkgrid\")","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"dir = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification'\n\ntrain_set1 = pd.read_csv(os.path.join(dir, 'jigsaw-toxic-comment-train.csv'))\ntrain_set2 = pd.read_csv(os.path.join(dir, 'jigsaw-unintended-bias-train.csv'))\ntrain_set2.toxic = train_set2.toxic.round().astype(int)\n\nvalid = pd.read_csv(os.path.join(dir, 'validation.csv'))\ntest = pd.read_csv(os.path.join(dir, 'test.csv'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Combine train1 with a subset of train2\ntrain = pd.concat([\n    train_set1[['comment_text', 'toxic']],\n    train_set2[['comment_text', 'toxic']].query('toxic==1'),\n    train_set2[['comment_text', 'toxic']].query('toxic==0').sample(n=100000, random_state=0)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.shape)\ntrain.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid.shape)\nvalid.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.shape)\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(train.toxic.value_counts())\nsns.countplot(train.toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid.toxic.value_counts())\nsns.countplot(valid.toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(valid.lang.value_counts())\nsns.countplot(valid.lang)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(test.lang.value_counts())\nsns.countplot(test.lang)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_ax(rows=1, cols=2, size=7):\n    \"\"\"Return a Matplotlib Axes array to be used in\n    all visualizations in the notebook. Provide a\n    central point to control graph sizes.\n    \n    Adjust the size attribute to control how big to render images\n    \"\"\"\n    fig, ax = plt.subplots(rows, cols, figsize=(size*cols, size*rows))\n    return fig, ax","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = get_ax()\n\nsns.distplot(train[train['toxic']==0]['comment_text'].str.len(), axlabel=\"Non Toxic\", ax=ax[0])\nsns.distplot(train[train['toxic']==0]['comment_text'].str.split().str.len(), axlabel=\"Non Toxic\", ax=ax[1])\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig, ax = get_ax()\n\nsns.distplot(train[train['toxic']==1]['comment_text'].str.len(), axlabel=\"Toxic\", ax=ax[0])\nsns.distplot(train[train['toxic']==1]['comment_text'].str.split().str.len(), axlabel=\"Toxic\", ax=ax[1])\n\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"stopwords = set(STOPWORDS)\n\ndef show_wordcloud(data, title = None):\n    wordcloud = WordCloud(\n        background_color='white',\n        stopwords=stopwords,\n        max_words=50,\n        max_font_size=40, \n        scale=5,\n        random_state=1\n    ).generate(str(data))\n\n    fig = plt.figure(1, figsize=(10,10))\n    plt.axis('off')\n    if title: \n        fig.suptitle(title, fontsize=20)\n        fig.subplots_adjust(top=2.3)\n\n    plt.imshow(wordcloud)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_wordcloud(train['comment_text'].sample(20000), \n               title = '[Comment_Text] Prevalent Words')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_wordcloud(valid['comment_text'].sample(1000), \n               title = '[Comment_Text] Prevalent Words')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"show_wordcloud(test['content'].sample(1000), \n               title = '[Content] Prevalent Words')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for i in range(5):\n    print(f'[CONTENT {i}]\\n', train['comment_text'][i])\n    print()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# fast encoder\ndef fast_encode(texts, tokenizer, chunk_size=240, maxlen=512):\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in range(0, len(texts), chunk_size):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# general encoder\ndef regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\n# Create strategy from tpu\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\n# Configuration\nEPOCHS = 4\nBATCH_SIZE = 16* strategy.num_replicas_in_sync\nMODEL = 'jplu/tf-xlm-roberta-large'\nMAX_LEN = 224","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# https://huggingface.co/models\nimport transformers\nfrom transformers import TFAutoModel, AutoTokenizer\nfrom tokenizers import Tokenizer, models, pre_tokenizers, decoders, processors\n\n#First load the real tokenizer\ntokenizer = AutoTokenizer.from_pretrained(MODEL)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nx_train = regular_encode(train.comment_text.values, \n                         tokenizer, maxlen=MAX_LEN)\nx_valid = regular_encode(valid.comment_text.values, \n                         tokenizer, maxlen=MAX_LEN)\nx_test = regular_encode(test.content.values, tokenizer, \n                        maxlen=MAX_LEN)\n\ny_train = train.toxic.values\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\ndef label_smoothing(y_true,y_pred):\n     return tf.keras.losses.binary_crossentropy(y_true,y_pred,label_smoothing=0.15)\n\ndef focal_loss(gamma=2., alpha=.2):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    \n    cls_token = sequence_output[:, 0, :]\n    x = Dropout(0.3)(cls_token)\n    out = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss=label_smoothing,\n                  metrics=[tf.keras.metrics.AUC()]) # competition metrics\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow.keras.backend as K\n\nwith strategy.scope():\n    transformer_layer = TFAutoModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer, max_len=MAX_LEN)\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from IPython.display import SVG\nSVG(tf.keras.utils.model_to_dot(model, dpi=70).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def callback():\n    cb = []\n    \"\"\"\n    Model-Checkpoint\n    \"\"\"\n    checkpoint = callbacks.ModelCheckpoint('model.h5',\n                                       save_best_only=True, \n                                       mode='min',\n                                       monitor='val_loss', #  \n                                       save_weights_only=True, verbose=0)\n\n    cb.append(checkpoint)\n    \n    # Callback that streams epoch results to a csv file.\n    log = callbacks.CSVLogger('log.csv')\n    cb.append(log)\n\n    return cb","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"calls = callback()\nn_steps = x_train.shape[0] // BATCH_SIZE\n\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    callbacks = calls,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def visualize_model_preds(model, indices=[0, 17, 1, 24]):\n    comments = valid.comment_text.loc[indices].values.tolist()\n    preds = model.predict(x_valid[indices].reshape(len(indices), -1))\n\n    for idx, i in enumerate(indices):\n        if y_valid[i] == 0:\n            label = \"Non-toxic\"\n            color = f'{Fore.GREEN}'\n            symbol = '\\u2714'\n        else:\n            label = \"Toxic\"\n            color = f'{Fore.RED}'\n            symbol = '\\u2716'\n\n        print('{}{} {}'.format(color, str(idx+1) + \". \" + label, symbol))\n        print(f'{Style.RESET_ALL}')\n        print(\"ORIGINAL\")\n        print(comments[idx]); print(\"\")\n        print(\"TRANSLATED\")\n        print(translator.translate(comments[idx]).text)\n        fig = go.Figure()\n        if list.index(sorted(preds[:, 0]), preds[idx][0]) > 1:\n            yl = [preds[idx][0], 1 - preds[idx][0]]\n        else:\n            yl = [1 - preds[idx][0], preds[idx][0]]\n        fig.add_trace(go.Bar(x=['Non-Toxic', 'Toxic'], y=yl, marker=dict(color=[\"seagreen\", \"indianred\"])))\n        fig.update_traces(name=comments[idx])\n        fig.update_layout(xaxis_title=\"Labels\", yaxis_title=\"Probability\", template=\"plotly_white\", title_text=\"Predictions for validation comment #{}\".format(idx+1))\n        fig.show()\n        \nvisualize_model_preds(model)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(dir + '/sample_submission.csv')\nsub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"References:Jigsaw Multilingual: Quick EDA & TPU Modeling by @Innat","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}