{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Introduction","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**In this kernel i will try to share my understanding and findings of cross lingual models.Feel free to correct me if I made any mistakes in this kernel.**\n\n","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Imports**","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n\nimport gc\nimport os\nimport time\nimport math\nimport random\nimport warnings\nimport numpy as np\nimport pandas as pd\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nfrom datetime import date\nfrom transformers import *\nfrom sklearn.metrics import *\nfrom tqdm.notebook import tqdm\n\nimport torch\nimport torch.nn as nn\nimport torch.utils.data\nimport torch.nn.functional as F\n\n\n\nimport warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport gc\nimport re\nimport folium\n#import textstat\nfrom scipy import stats\nfrom colorama import Fore, Back, Style, init\n\nimport math\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\n\nimport random\nimport networkx as nx\nfrom pandas import Timestamp\n\nfrom PIL import Image\nfrom IPython.display import SVG\nfrom keras.utils import model_to_dot\n\nimport requests\nfrom IPython.display import HTML\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\n\ntqdm.pandas()\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\nimport transformers\nimport tensorflow as tf\n\nfrom tensorflow.keras.callbacks import Callback\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\n\nfrom tensorflow.keras.models import Model\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.optimizers import Adam\nfrom tokenizers import BertWordPieceTokenizer\nfrom tensorflow.keras.layers import Dense, Input, Dropout, Embedding\nfrom tensorflow.keras.layers import LSTM, GRU, Conv1D, SpatialDropout1D\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import activations\nfrom tensorflow.keras import constraints\nfrom tensorflow.keras import initializers\nfrom tensorflow.keras import regularizers\n\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.activations import *\nfrom tensorflow.keras.constraints import *\nfrom tensorflow.keras.initializers import *\nfrom tensorflow.keras.regularizers import *\n\nfrom sklearn import metrics\nfrom sklearn.utils import shuffle\nfrom gensim.models import Word2Vec\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_extraction.text import TfidfVectorizer,\\\n                                            CountVectorizer,\\\n                                            HashingVectorizer\n\nfrom nltk.stem.wordnet import WordNetLemmatizer \nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer  \n\nimport nltk\nfrom textblob import TextBlob\n\nfrom nltk.corpus import wordnet\nfrom nltk.corpus import stopwords\n#from googletrans import Translator\nfrom nltk import WordNetLemmatizer\n#from polyglot.detect import Detector\nfrom nltk.stem import WordNetLemmatizer\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\n\nstopword=set(STOPWORDS)\n\nlem = WordNetLemmatizer()\ntokenizer=TweetTokenizer()\n\nnp.random.seed(0)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Setup TPU configuration","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\nprint(strategy.num_replicas_in_sync)\n#GCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Part 2 : Implementation using TPU Multiprocessing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\n# train2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n# train2.toxic = train2.toxic.round().astype(int)\n\n# train3 = pd.read_csv('../input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')\n\n# train4 = pd.read_csv('../input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')\n\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\n# test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\n# sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\n\n# toxic = len(train2[['comment_text', 'toxic']].query('toxic==1'))\n# # Combine train1 with a subset of train2\n# train = pd.concat([\n#     train1[['comment_text', 'toxic']],\n#     train2[['comment_text', 'toxic']].query('toxic==1'),\n    \n#     train3[['comment_text', 'toxic']].query('toxic==0'),\n#     train3[['comment_text', 'toxic']].query('toxic==1'),\n    \n#     train4[['comment_text', 'toxic']].query('toxic==0'),\n#     train4[['comment_text', 'toxic']].query('toxic==1'),\n    \n    \n#     train2[['comment_text', 'toxic']].query('toxic==0').sample(n=(toxic+(toxic//3)), random_state=101)\n# ])\n\n# test_data = test\n# train_data = train\n\nmaxlen = 192","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# valid.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# import matplotlib.pyplot as plt\n# import seaborn as sns\n# plt.figure(figsize=(10, 5))\n# plt.subplot(1, 2, 1)\n# sns.countplot(train_data['toxic'])\n# plt.title('Target on training data')\n\n# plt.subplot(1, 2, 2)\n# sns.countplot(valid['toxic'])\n# plt.title('Target on validation data')\n\n# plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# df_valid_en = pd.read_csv('../input/val-en-df/validation_en.csv')\n# df_valid_en","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":false},"cell_type":"code","source":"\n\n\n\n# df_valid_en = df_valid_en.drop(columns=['id', 'comment_text','lang'])\n# df_valid_en = df_valid_en.rename(columns={\"comment_text_en\": \"comment_text\"})\n# columns_titles = [\"comment_text\",\"toxic\"]\n# df_valid_en=df_valid_en.reindex(columns=columns_titles)\n\n# '''df2 = df_valid_en[df_valid_en.toxic == 1]\n# df_valid_en = pd.concat([df_valid_en,df2])\n# df_valid_en = pd.concat([df_valid_en,df2])  #doubling twice\n# '''\n\n# print(df_valid_en.head(3))\n\n# train_data = pd.concat([train_data , df_valid_en], axis=0).reset_index(drop=True)\n# train_data = train_data.sample(frac=1).reset_index(drop=True)\n\n# print(len(train2[['comment_text', 'toxic']].query('toxic==1')))\n# print(len(train_data))\n# train_data.head()\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Clean the text (remove usernames and links)**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# val = valid\n# train = train_data\n\n# def clean(text):\n#     text = text.fillna(\"fillna\").str.lower()\n#     text = text.map(lambda x: re.sub('\\\\n',' ',str(x)))\n#     text = text.map(lambda x: re.sub(\"\\[\\[User.*\",'',str(x)))\n#     text = text.map(lambda x: re.sub(\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",'',str(x)))\n#     text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n#     return text\n\n# val[\"comment_text\"] = clean(val[\"comment_text\"])\n# test_data[\"content\"] = clean(test_data[\"content\"])\n# train[\"comment_text\"] = clean(train[\"comment_text\"])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# More Text Cleaning\n\n**applying text cleaning techniques like clean_text,replace_typical_misspell,handle_contractions,fix_quote \non train,test and validation set**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# # https://www.kaggle.com/chenshengabc/from-quest-encoding-ensemble-a-little-bit-differen\n\n# puncts = [',', '.', '\"', ':', ')', '(', '-', '!', '?', '|', ';', \"'\", '$', '&', '/', '[', ']', '>', '%', '=', '#', '*', '+', '\\\\', '•',  '~', '@', '£',\n#  '·', '_', '{', '}', '©', '^', '®', '`',  '<', '→', '°', '€', '™', '›',  '♥', '←', '×', '§', '″', '′', 'Â', '█', '½', 'à', '…', '\\xa0', '\\t',\n#  '“', '★', '”', '–', '●', 'â', '►', '−', '¢', '²', '¬', '░', '¶', '↑', '±', '¿', '▾', '═', '¦', '║', '―', '¥', '▓', '—', '‹', '─', '\\u3000', '\\u202f',\n#  '▒', '：', '¼', '⊕', '▼', '▪', '†', '■', '’', '▀', '¨', '▄', '♫', '☆', 'é', '¯', '♦', '¤', '▲', 'è', '¸', '¾', 'Ã', '⋅', '‘', '∞', '«',\n#  '∙', '）', '↓', '、', '│', '（', '»', '，', '♪', '╩', '╚', '³', '・', '╦', '╣', '╔', '╗', '▬', '❤', 'ï', 'Ø', '¹', '≤', '‡', '√', ]\n\n# mispell_dict = {\"aren't\" : \"are not\",\n# \"can't\" : \"cannot\",\n# \"couldn't\" : \"could not\",\n# \"couldnt\" : \"could not\",\n# \"didn't\" : \"did not\",\n# \"doesn't\" : \"does not\",\n# \"doesnt\" : \"does not\",\n# \"don't\" : \"do not\",\n# \"hadn't\" : \"had not\",\n# \"hasn't\" : \"has not\",\n# \"haven't\" : \"have not\",\n# \"havent\" : \"have not\",\n# \"he'd\" : \"he would\",\n# \"he'll\" : \"he will\",\n# \"he's\" : \"he is\",\n# \"i'd\" : \"I would\",\n# \"i'd\" : \"I had\",\n# \"i'll\" : \"I will\",\n# \"i'm\" : \"I am\",\n# \"isn't\" : \"is not\",\n# \"it's\" : \"it is\",\n# \"it'll\":\"it will\",\n# \"i've\" : \"I have\",\n# \"let's\" : \"let us\",\n# \"mightn't\" : \"might not\",\n# \"mustn't\" : \"must not\",\n# \"shan't\" : \"shall not\",\n# \"she'd\" : \"she would\",\n# \"she'll\" : \"she will\",\n# \"she's\" : \"she is\",\n# \"shouldn't\" : \"should not\",\n# \"shouldnt\" : \"should not\",\n# \"that's\" : \"that is\",\n# \"thats\" : \"that is\",\n# \"there's\" : \"there is\",\n# \"theres\" : \"there is\",\n# \"they'd\" : \"they would\",\n# \"they'll\" : \"they will\",\n# \"they're\" : \"they are\",\n# \"theyre\":  \"they are\",\n# \"they've\" : \"they have\",\n# \"we'd\" : \"we would\",\n# \"we're\" : \"we are\",\n# \"weren't\" : \"were not\",\n# \"we've\" : \"we have\",\n# \"what'll\" : \"what will\",\n# \"what're\" : \"what are\",\n# \"what's\" : \"what is\",\n# \"what've\" : \"what have\",\n# \"where's\" : \"where is\",\n# \"who'd\" : \"who would\",\n# \"who'll\" : \"who will\",\n# \"who're\" : \"who are\",\n# \"who's\" : \"who is\",\n# \"who've\" : \"who have\",\n# \"won't\" : \"will not\",\n# \"wouldn't\" : \"would not\",\n# \"you'd\" : \"you would\",\n# \"you'll\" : \"you will\",\n# \"you're\" : \"you are\",\n# \"you've\" : \"you have\",\n# \"'re\": \" are\",\n# \"wasn't\": \"was not\",\n# \"we'll\":\" will\",\n# \"didn't\": \"did not\",\n# \"tryin'\":\"trying\"}\n\n\n# def clean_text(x):\n#     x = str(x).replace(\"\\n\",\"\")\n#     for punct in puncts:\n#         x = x.replace(punct, f' {punct} ')\n#     return x\n\n\n# def clean_numbers(x):\n#     x = re.sub('[0-9]{5,}', '#####', x)\n#     x = re.sub('[0-9]{4}', '####', x)\n#     x = re.sub('[0-9]{3}', '###', x)\n#     x = re.sub('[0-9]{2}', '##', x)\n#     return x","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from nltk.tokenize.treebank import TreebankWordTokenizer\n# tokenizer = TreebankWordTokenizer()\n\n# def handle_contractions(x):\n#     x = tokenizer.tokenize(x)\n#     return x\n\n# def fix_quote(x):\n#     x = [x_[1:] if x_.startswith(\"'\") else x_ for x_ in x]\n#     x = ' '.join(x)\n#     return x\n\n# def _get_mispell(mispell_dict):\n#     mispell_re = re.compile('(%s)' % '|'.join(mispell_dict.keys()))\n#     return mispell_dict, mispell_re\n\n\n# def replace_typical_misspell(text):\n#     mispellings, mispellings_re = _get_mispell(mispell_dict)\n\n#     def replace(match):\n#         return mispellings[match.group(0)]\n\n#     return mispellings_re.sub(replace, text)\n\n\n# def clean_data(df, columns: list):\n#     for col in columns:\n# #         df[col] = df[col].apply(lambda x: clean_numbers(x))\n#         df[col] = df[col].apply(lambda x: clean_text(x.lower())) \n#         df[col] = df[col].apply(lambda x: replace_typical_misspell(x))\n#         df[col] = df[col].apply(lambda x: handle_contractions(x))  \n#         df[col] = df[col].apply(lambda x: fix_quote(x))   \n    \n#     return df\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# input_columns = [\n#     'comment_text'   \n# ]\n\n# '''applying text cleaning techniques like clean_text,replace_typical_misspell,handle_contractions,fix_quote \n# on train,test and validation set'''\n\n# train = clean_data(train, input_columns ) \n# val = clean_data(val, input_columns )\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# input_columns = [\n#     'content'   \n# ]\n# test_data = clean_data(test_data, input_columns )\n\n# del tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# val.to_csv('/kaggle/working/valid_cleaned.csv')\n# test_data.to_csv('/kaggle/working/test_cleaned.csv')\n# train.to_csv('/kaggle/working/train_cleaned.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# val = pd.read_csv('/kaggle/working/valid_cleaned.csv')\n# test_data = pd.read_csv('/kaggle/working/test_cleaned.csv')\n# train = pd.read_csv('/kaggle/working/train_cleaned.csv')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**we can see from above 2 cells that text cleaning for train,validation and test set takes 8+ minutes that means we are losing some of our vital times for training  on tpu which is 3 hours(max). so it would be a good idea if we create another kernel and save above 2 cells newly updated train,val and test_data as kernels output then using those files we can quickly import our new train,test and validation data here,which will save time for training model on TPU **","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"**Roc-Auc Evaluation metric**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class RocAucEvaluation(Callback):\n    def __init__(self, validation_data=(), interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.X_val, self.y_val = validation_data\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict(self.X_val, verbose=0)\n            score = roc_auc_score(self.y_val, y_pred)\n            print(\"\\n ROC-AUC - epoch: {:d} - score: {:.6f}\".format(epoch+1, score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenize(encode) comments","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(enc_di['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Load bert tokenizer**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL = 'jplu/tf-xlm-roberta-large'\n# First load the real tokenizer\ntokenizer = AutoTokenizer.from_pretrained(MODEL)\n\nsave_path = '/kaggle/working/xlmr_large/'\nif not os.path.exists(save_path):\n    os.makedirs(save_path)\ntokenizer.save_pretrained(save_path)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Encode comments**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# %%time\n# x_train = regular_encode(train.comment_text.astype(str), \n#                       tokenizer, maxlen=maxlen)\n# x_valid = regular_encode(val.comment_text.astype(str).values, \n#                       tokenizer, maxlen=maxlen)\n# x_test = regular_encode(test_data.content.astype(str).values, \n#                      tokenizer, maxlen=maxlen)\n\n# y_valid = val.toxic.values\n# y_train = train.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# np.save('/kaggle/working/xtrain.npy', x_train)\n# np.save('/kaggle/working/ytrain.npy', y_train)\n# np.save('/kaggle/working/xtest.npy', x_test)\n# np.save('/kaggle/working/xval.npy', x_valid)\n# np.save('/kaggle/working/yval.npy', y_valid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = np.load('/kaggle/input/kernel9/drive-download-20200522T103552Z-001/xtrain_9.npy')\nx_test = np.load('/kaggle/input/kernel9/drive-download-20200522T103552Z-001/xtest_9.npy')\nx_valid = np.load('/kaggle/input/kernel9/drive-download-20200522T103552Z-001/xval_9.npy')\ny_train = np.load('/kaggle/input/kernel9/drive-download-20200522T103552Z-001/ytrain_9.npy')\ny_valid = np.load('/kaggle/input/kernel9/drive-download-20200522T103552Z-001/yval_9.npy')\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**previously we lost 8+ minutes and here ~12 minutes,sum them and we have lost 20+ minutes, which is almost 1 epoch training time here..!! :(**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"MODEL","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Train. validation and testing dataset**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ntrain_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Focal Loss","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras import backend as K\n\ndef focal_loss(gamma=2., alpha=.2):\n    def focal_loss_fixed(y_true, y_pred):\n        pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n        pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n        return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n    return focal_loss_fixed","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Build the model and check summary","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, loss='binary_crossentropy', max_len=512):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    x = tf.keras.layers.Dropout(0.3)(cls_token)\n    out = Dense(1, activation='sigmoid')(x)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC()])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = transformers.TFXLMRobertaModel.from_pretrained(MODEL)\n    model = build_model(transformer_layer,loss='binary_crossentropy', max_len=maxlen)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Define Define ReduceLROnPlateau callback**","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def callback():\n    cb = []\n\n    reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss',  \n                                    factor=0.3, patience=2, \n                                    verbose=1, mode='auto', \n                                    epsilon=0.0001, cooldown=1, min_lr=0.000001)\n    cb.append(reduceLROnPlat)\n    log = CSVLogger('log.csv')\n    cb.append(log)\n\n    RocAuc = RocAucEvaluation(validation_data=(x_valid, y_valid), interval=1)\n    cb.append(RocAuc)\n    \n    return cb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Visualize model architecture","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"SVG(tf.keras.utils.model_to_dot(model, dpi=70).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Learning rate schedule","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lrfn(lr_start=0.000001, lr_max=0.000002, \n               lr_min=0.0000001, lr_rampup_epochs=7, \n               lr_sustain_epochs=0, lr_exp_decay=.87):\n    lr_max = lr_max * strategy.num_replicas_in_sync\n\n    def lrfn(epoch):\n        if epoch < lr_rampup_epochs:\n            lr = (lr_max - lr_start) / lr_rampup_epochs * epoch + lr_start\n        elif epoch < lr_rampup_epochs + lr_sustain_epochs:\n            lr = lr_max\n        else:\n            lr = (lr_max - lr_min) * lr_exp_decay**(epoch - lr_rampup_epochs - lr_sustain_epochs) + lr_min\n        return lr\n    \n    return lrfn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nplt.figure(figsize=(10, 7))\n\nlrfn = build_lrfn()\nplt.plot([i for i in range(35)], [lrfn(i) for i in range(35)]);","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model_path = 'jigsawMultilingual.hdf5'\nmodel_path1 = '/kaggle/working/jigsawMultilingual.hdf5'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n\ncheckpoint = ModelCheckpoint(model_path, monitor='val_accuracy', mode='max', save_best_only=True)\nes = EarlyStopping(monitor='val_accuracy', mode='max', patience=2, \n                   restore_best_weights=True, verbose=1)\nlr_callback = LearningRateScheduler(lrfn, verbose=1)\n\ncallback_list = [checkpoint,  lr_callback]","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Training","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nN_STEPS = x_train.shape[0] // BATCH_SIZE\nEPOCHS = 2\ntrain_history = model.fit(\n    train_dataset,\n    steps_per_epoch=N_STEPS,\n    validation_data=valid_dataset,\n    callbacks=callback_list,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"v=valid.copy()[['id','toxic']]\nv['toxic'] = model.predict(valid_dataset, verbose=1)\nv.to_csv('understanding_cross_lingual_models_submission_val.csv', index=False)\nprint(roc_auc_score(valid['toxic'], v['toxic']))\nv","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.exists(model_path1):\n    model.load_weights(model_path1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"we train it for 2 more epochs on the validation set, which is significantly smaller but contains a mixture of different languages.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nn_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    callbacks=callback_list,\n    epochs= EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"if os.path.exists(model_path1):\n    model.load_weights(model_path1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"log_dir = \"/kaggle/working/log.csv\"\nif os.path.exists(log_dir):\n    os.remove(log_dir)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make Submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/' + 'sample_submission.csv')\nsub['toxic'] = model.predict(test_dataset, verbose=1)\nsub.to_csv('understanding_cross_lingual_models_submission_test.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**References**\n* [Jigsaw TPU: XLM-Roberta](https://www.kaggle.com/xhlulu/jigsaw-tpu-xlm-roberta)\n* [Jigsaw TPU: DistilBERT with Huggingface and Keras](https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras)\n* [Jigsaw TPU: BERT with Huggingface and Keras](https://www.kaggle.com/miklgr500/jigsaw-tpu-bert-with-huggingface-and-keras)\n* [8 Excellent Pretrained Models to get you Started with Natural Language Processing (NLP)](https://www.analyticsvidhya.com/blog/2019/03/pretrained-models-get-started-nlp/)\n* [Introduction to PyTorch-Transformers: An Incredible Library for State-of-the-Art NLP (with Python code)](https://www.analyticsvidhya.com/blog/2019/07/pytorch-transformers-nlp-python/)\n\n* [facebookresearch/XLM](https://github.com/facebookresearch/XLM)\n\n* [The Illustrated Transformer](https://jalammar.github.io/illustrated-transformer/)\n\n* [Exploring Distributional Representations and Machine Translation for Aspect-based Cross-lingual Sentiment Classification](https://www.researchgate.net/publication/309312650_Exploring_Distributional_Representations_and_Machine_Translation_for_Aspect-based_Cross-lingual_Sentiment_Classification)\n* [Fastai with 🤗 Transformers (BERT, RoBERTa, ...)](https://www.kaggle.com/maroberti/fastai-with-transformers-bert-roberta)\n* [Jigsaw Multilingual Toxicity : EDA + Models](https://www.kaggle.com/tarunpaparaju/jigsaw-multilingual-toxicity-eda-models)\n* [Flower Classification with TPUs - EDA and Baseline](https://www.kaggle.com/dimitreoliveira/flower-classification-with-tpus-eda-and-baseline/notebook)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}