{"cells":[{"metadata":{},"cell_type":"markdown","source":"# XLM-Roberta Tokenizer [Jigsaw Toxic Comment]","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"I split training and tokenization into two seperate notebooks and later chained them together. This conserved TPU time and it also left more resources for training.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport re\nimport numpy as np\nimport pandas as pd\nimport transformers\nimport tensorflow as tf\nimport tqdm.notebook as tqdm\nimport matplotlib.pyplot as plt\nfrom tensorflow.keras.layers import Dense, Input, Activation\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.optimizers import Adam","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper Functions\nThrough experiments, I found that cleaning the data (getting rid of usernames, ip addresses, removing symbols) does not improve model score. In some cases, it even diminished it.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts,\n        return_attention_masks=False,\n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"print(os.listdir('/kaggle/input/jigsaw-multilingual-toxic-comment-classification'))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"eng = '/kaggle/input/jigsaw-multilingual-toxic-comment-classification/'\nnon_eng = '/kaggle/input/jigsaw-train-multilingual-coments-google-api/'\n\nes = pd.read_csv(f'{non_eng}jigsaw-toxic-comment-train-google-es-cleaned.csv')\nfr = pd.read_csv(f'{non_eng}jigsaw-toxic-comment-train-google-fr-cleaned.csv')\nit = pd.read_csv(f'{non_eng}jigsaw-toxic-comment-train-google-it-cleaned.csv')\npt = pd.read_csv(f'{non_eng}jigsaw-toxic-comment-train-google-pt-cleaned.csv')\nru = pd.read_csv(f'{non_eng}jigsaw-toxic-comment-train-google-ru-cleaned.csv')\ntr = pd.read_csv(f'{non_eng}jigsaw-toxic-comment-train-google-tr-cleaned.csv')\n\nfor df in [es, fr, it, pt, ru, tr]:\n    cols = list(df.columns)[3:]\n    df.toxic = df[cols].sum(axis=1)\n    df.toxic.apply(lambda x: 1 if x >= 1 else 0)\n\neg2 = pd.read_csv(f'{eng}jigsaw-unintended-bias-train.csv')\neg2['toxic'] = eg2.toxic.round().astype(int)\n\nvalid = pd.read_csv(f'{eng}validation.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"bias_augment = '/kaggle/input/translated-train-bias-all-langs/All languages'\nbias_es = pd.read_csv(f'{bias_augment}/train-bias-toxic-google-api-es-cleaned.csv')\nbias_fr = pd.read_csv(f'{bias_augment}/train-bias-toxic-google-api-fr-cleaned.csv')\nbias_it = pd.read_csv(f'{bias_augment}/train-bias-toxic-google-api-it-cleaned.csv')\nbias_pt = pd.read_csv(f'{bias_augment}/train-bias-toxic-google-api-pt-cleaned.csv')\nbias_ru = pd.read_csv(f'{bias_augment}/train-bias-toxic-google-api-ru-cleaned.csv')\nbias_tr = pd.read_csv(f'{bias_augment}/train-bias-toxic-google-api-tr-cleaned.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"### External Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_ex = '/kaggle/input/toxic-comment-detection-multilingual-extended/english/english/'\ntest1 = pd.read_csv(f'{eng}test.csv')\ntesten2 = pd.read_csv(f'{test_ex}test_en.csv')\ntesten3 = pd.read_csv(f'{test_ex}jigsaw_miltilingual_test_translated.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"external = '/kaggle/input/toxic-comment-detection-multilingual-extended/archive/'\ne_ru = pd.read_csv(f'{external}russian/labeled.csv')\n\ne_tr = pd.read_csv(f'{external}turkish/troff-v1.0.tsv', sep='\\t', header=0)\ne_tr.label = e_tr.label.apply(lambda x: 1 if x not in ['non', 'prof'] else 0)\n\ne_it = pd.concat([\n    pd.read_csv(f'{external}italian/haspeede_FB-train.tsv', sep='\\t', header=0),\n    pd.read_csv(f'{external}italian/haspeede_TW-train.tsv', sep='\\t', header=0)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"e_ru.rename(columns={'comment':'comment_text'}, inplace=True)\ne_tr.rename(columns={'text':'comment_text', 'label':'toxic'}, inplace=True)\ne_it.rename(columns={'comment':'comment_text'}, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat([\n    es[['comment_text', 'toxic']].query('toxic==0').sample(n=20000, random_state=0),\n    es[['comment_text', 'toxic']].query('toxic==1'),\n    fr[['comment_text', 'toxic']].query('toxic==0').sample(n=20000, random_state=0),\n    fr[['comment_text', 'toxic']].query('toxic==1'),\n    it[['comment_text', 'toxic']].query('toxic==0').sample(n=20000, random_state=0),\n    it[['comment_text', 'toxic']].query('toxic==1'),\n    pt[['comment_text', 'toxic']].query('toxic==0').sample(n=20000, random_state=0),\n    pt[['comment_text', 'toxic']].query('toxic==1'),\n    ru[['comment_text', 'toxic']].query('toxic==0').sample(n=20000, random_state=0),\n    ru[['comment_text', 'toxic']].query('toxic==1'),\n    tr[['comment_text', 'toxic']].query('toxic==0').sample(n=20000, random_state=0),\n    tr[['comment_text', 'toxic']].query('toxic==1'),\n    eg2[['comment_text', 'toxic']].query('toxic==1'),\n    eg2[['comment_text', 'toxic']].query('toxic==0').sample(n=200000, random_state=0),\n    bias_es[['comment_text', 'toxic']].query('toxic==1'),\n    bias_fr[['comment_text', 'toxic']].query('toxic==1'),\n    bias_it[['comment_text', 'toxic']].query('toxic==1'),\n    bias_pt[['comment_text', 'toxic']].query('toxic==1'),\n    bias_ru[['comment_text', 'toxic']].query('toxic==1'),\n    bias_tr[['comment_text', 'toxic']].query('toxic==1')\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.toxic = train.toxic.round().astype(int)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Data Visualizations","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = plt.figure()\nax = fig.add_axes([0, 0, 1, 1])\nax.set_title(f'Count of Toxic and Non-Toxic Training Datapoints: {len(train)}')\nc_toxic, c_nontoxic = len(train[train['toxic']==1]), len(train[train['toxic']==0])\nlabels = [f'Toxic: {c_toxic}', f'Non-Toxic {c_nontoxic}']\nvalues = [c_toxic, c_nontoxic]\nax.bar(labels, values)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenization\nUsing XLM-Roberta (Large)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nx_train = encode(train.comment_text.values, tokenizer, maxlen=192)\nx_valid = encode(valid.comment_text.values, tokenizer, maxlen=192)\nx_test1 = encode(test1.content.values, tokenizer, maxlen=192)\nx_test2 = encode(testen2.content_en, tokenizer, maxlen=192)\nx_test3 = encode(testen3.translated, tokenizer, maxlen=192)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('x_train', x_train)\nnp.save('x_valid', x_valid)\nnp.save('x_test1', x_test1)\nnp.save('x_test2', x_test2)\nnp.save('x_test3', x_test3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.save('y_train', train.toxic.values)\nnp.save('y_valid', valid.toxic.values)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}