{"cells":[{"metadata":{"_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","trusted":false},"cell_type":"code","source":"from pathlib import Path\nimport numpy as np \nimport pandas as pd \nimport seaborn as sns\nfrom matplotlib import pyplot as plt\n%matplotlib inline\nimport re\nimport tensorflow as tf\nimport transformers\nfrom tensorflow.keras.layers import Dense, Input, Dropout\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras import regularizers\nimport warnings\n\nwarnings.filterwarnings(\"ignore\")\n#from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, ReduceLROnPlateau","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"DATA_ROOT = Path(\"..\")/\"input\"/ \"jigsaw-multilingual-toxic-comment-classification/\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df1,df2,df3 = [pd.read_csv(DATA_ROOT / fname, usecols=[\"comment_text\", \"toxic\"]) for fname in [\"jigsaw-toxic-comment-train.csv\",\n                                                            \"jigsaw-unintended-bias-train.csv\",\n                                                            \"validation.csv\"\n                                                                       ]]\n\n\n\ntest, sample= [pd.read_csv(DATA_ROOT / fname) for fname in [\"test.csv\",\n                                                            \"sample_submission.csv\"\n                                                            ]]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# #Clean up the comment text\n# def clean_text(text):\n#     text = text.lower()\n    \n#     text = re.sub(r\"what's\", \"what is \", text)\n#     text = re.sub(r\"\\'s\", \" \", text)\n#     text = re.sub(r\"\\'ve\", \" have \", text)\n#     text = re.sub(r\"can't\", \"cannot \", text)\n#     text = re.sub(r\"n't\", \" not \", text)\n#     text = re.sub(r\"i'm\", \"i am \", text)\n#     text = re.sub(r\"\\'re\", \" are \", text)\n#     text = re.sub(r\"\\'d\", \" would \", text)\n#     text = re.sub(r\"\\'ll\", \" will \", text)\n#     text = re.sub(r\"\\'scuse\", \" excuse \", text)\n#     text = re.sub('\\W', ' ', text)\n#     text = re.sub('\\s+', ' ', text)\n#     text = text.strip(' ')\n#     return text","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# #Clean up the comment text\n# def clean_text(g_text):\n    \n#     # Special characters\n#     g_text = re.sub(r\"\\x89Û_\", \"\", g_text)\n#     g_text = re.sub(r\"\\x89ÛÒ\", \"\", g_text)\n#     g_text = re.sub(r\"\\x89ÛÓ\", \"\", g_text)\n#     g_text = re.sub(r\"\\x89ÛÏWhen\", \"When\", g_text)\n#     g_text = re.sub(r\"\\x89ÛÏ\", \"\", g_text)\n#     g_text = re.sub(r\"China\\x89Ûªs\", \"China's\", g_text)\n#     g_text = re.sub(r\"let\\x89Ûªs\", \"let's\", g_text)\n#     g_text = re.sub(r\"\\x89Û÷\", \"\", g_text)\n#     g_text = re.sub(r\"\\x89Ûª\", \"\", g_text)\n#     g_text = re.sub(r\"\\x89Û\\x9d\", \"\", g_text)\n#     g_text = re.sub(r\"å_\", \"\", g_text)\n#     g_text = re.sub(r\"\\x89Û¢\", \"\", g_text)\n#     g_text = re.sub(r\"\\x89Û¢åÊ\", \"\", g_text)\n#     g_text = re.sub(r\"fromåÊwounds\", \"from wounds\", g_text)\n#     g_text = re.sub(r\"åÊ\", \"\", g_text)\n#     g_text = re.sub(r\"åÈ\", \"\", g_text)\n#     g_text = re.sub(r\"JapÌ_n\", \"Japan\", g_text)    \n#     g_text = re.sub(r\"Ì©\", \"e\", g_text)\n#     g_text = re.sub(r\"å¨\", \"\", g_text)\n#     g_text = re.sub(r\"Surṳ\", \"Suruc\", g_text)\n#     g_text = re.sub(r\"åÇ\", \"\", g_text)\n#     g_text = re.sub(r\"å£3million\", \"3 million\", g_text)\n#     g_text = re.sub(r\"åÀ\", \"\", g_text)\n    \n#     # Contractions\n#     g_text = re.sub(r\"he's\", \"he is\", g_text)\n#     g_text = re.sub(r\"there's\", \"there is\", g_text)\n#     g_text = re.sub(r\"We're\", \"We are\", g_text)\n#     g_text = re.sub(r\"That's\", \"That is\", g_text)\n#     g_text = re.sub(r\"won't\", \"will not\", g_text)\n#     g_text = re.sub(r\"they're\", \"they are\", g_text)\n#     g_text = re.sub(r\"Can't\", \"Cannot\", g_text)\n#     g_text = re.sub(r\"wasn't\", \"was not\", g_text)\n#     g_text = re.sub(r\"don\\x89Ûªt\", \"do not\", g_text)\n#     g_text = re.sub(r\"aren't\", \"are not\", g_text)\n#     g_text = re.sub(r\"isn't\", \"is not\", g_text)\n#     g_text = re.sub(r\"What's\", \"What is\", g_text)\n#     g_text = re.sub(r\"haven't\", \"have not\", g_text)\n#     g_text = re.sub(r\"hasn't\", \"has not\", g_text)\n#     g_text = re.sub(r\"There's\", \"There is\", g_text)\n#     g_text = re.sub(r\"He's\", \"He is\", g_text)\n#     g_text = re.sub(r\"It's\", \"It is\", g_text)\n#     g_text = re.sub(r\"You're\", \"You are\", g_text)\n#     g_text = re.sub(r\"I'M\", \"I am\", g_text)\n#     g_text = re.sub(r\"shouldn't\", \"should not\", g_text)\n#     g_text = re.sub(r\"wouldn't\", \"would not\", g_text)\n#     g_text = re.sub(r\"i'm\", \"I am\", g_text)\n#     g_text = re.sub(r\"I\\x89Ûªm\", \"I am\", g_text)\n#     g_text = re.sub(r\"I'm\", \"I am\", g_text)\n#     g_text = re.sub(r\"Isn't\", \"is not\", g_text)\n#     g_text = re.sub(r\"Here's\", \"Here is\", g_text)\n#     g_text = re.sub(r\"you've\", \"you have\", g_text)\n#     g_text = re.sub(r\"you\\x89Ûªve\", \"you have\", g_text)\n#     g_text = re.sub(r\"we're\", \"we are\", g_text)\n#     g_text = re.sub(r\"what's\", \"what is\", g_text)\n#     g_text = re.sub(r\"couldn't\", \"could not\", g_text)\n#     g_text = re.sub(r\"we've\", \"we have\", g_text)\n#     g_text = re.sub(r\"it\\x89Ûªs\", \"it is\", g_text)\n#     g_text = re.sub(r\"doesn\\x89Ûªt\", \"does not\", g_text)\n#     g_text = re.sub(r\"It\\x89Ûªs\", \"It is\", g_text)\n#     g_text = re.sub(r\"Here\\x89Ûªs\", \"Here is\", g_text)\n#     g_text = re.sub(r\"who's\", \"who is\", g_text)\n#     g_text = re.sub(r\"I\\x89Ûªve\", \"I have\", g_text)\n#     g_text = re.sub(r\"y'all\", \"you all\", g_text)\n#     g_text = re.sub(r\"can\\x89Ûªt\", \"cannot\", g_text)\n#     g_text = re.sub(r\"would've\", \"would have\", g_text)\n#     g_text = re.sub(r\"it'll\", \"it will\", g_text)\n#     g_text = re.sub(r\"we'll\", \"we will\", g_text)\n#     g_text = re.sub(r\"wouldn\\x89Ûªt\", \"would not\", g_text)\n#     g_text = re.sub(r\"We've\", \"We have\", g_text)\n#     g_text = re.sub(r\"he'll\", \"he will\", g_text)\n#     g_text = re.sub(r\"Y'all\", \"You all\", g_text)\n#     g_text = re.sub(r\"Weren't\", \"Were not\", g_text)\n#     g_text = re.sub(r\"Didn't\", \"Did not\", g_text)\n#     g_text = re.sub(r\"they'll\", \"they will\", g_text)\n#     g_text = re.sub(r\"they'd\", \"they would\", g_text)\n#     g_text = re.sub(r\"DON'T\", \"DO NOT\", g_text)\n#     g_text = re.sub(r\"That\\x89Ûªs\", \"That is\", g_text)\n#     g_text = re.sub(r\"they've\", \"they have\", g_text)\n#     g_text = re.sub(r\"i'd\", \"I would\", g_text)\n#     g_text = re.sub(r\"should've\", \"should have\", g_text)\n#     g_text = re.sub(r\"You\\x89Ûªre\", \"You are\", g_text)\n#     g_text = re.sub(r\"where's\", \"where is\", g_text)\n#     g_text = re.sub(r\"Don\\x89Ûªt\", \"Do not\", g_text)\n#     g_text = re.sub(r\"we'd\", \"we would\", g_text)\n#     g_text = re.sub(r\"i'll\", \"I will\", g_text)\n#     g_text = re.sub(r\"weren't\", \"were not\", g_text)\n#     g_text = re.sub(r\"They're\", \"They are\", g_text)\n#     g_text = re.sub(r\"Can\\x89Ûªt\", \"Cannot\", g_text)\n#     g_text = re.sub(r\"you\\x89Ûªll\", \"you will\", g_text)\n#     g_text = re.sub(r\"I\\x89Ûªd\", \"I would\", g_text)\n#     g_text = re.sub(r\"let's\", \"let us\", g_text)\n#     g_text = re.sub(r\"it's\", \"it is\", g_text)\n#     g_text = re.sub(r\"can't\", \"cannot\", g_text)\n#     g_text = re.sub(r\"don't\", \"do not\", g_text)\n#     g_text = re.sub(r\"you're\", \"you are\", g_text)\n#     g_text = re.sub(r\"i've\", \"I have\", g_text)\n#     g_text = re.sub(r\"that's\", \"that is\", g_text)\n#     g_text = re.sub(r\"i'll\", \"I will\", g_text)\n#     g_text = re.sub(r\"doesn't\", \"does not\", g_text)\n#     g_text = re.sub(r\"i'd\", \"I would\", g_text)\n#     g_text = re.sub(r\"didn't\", \"did not\", g_text)\n#     g_text = re.sub(r\"ain't\", \"am not\", g_text)\n#     g_text = re.sub(r\"you'll\", \"you will\", g_text)\n#     g_text = re.sub(r\"I've\", \"I have\", g_text)\n#     g_text = re.sub(r\"Don't\", \"do not\", g_text)\n#     g_text = re.sub(r\"I'll\", \"I will\", g_text)\n#     g_text = re.sub(r\"I'd\", \"I would\", g_text)\n#     g_text = re.sub(r\"Let's\", \"Let us\", g_text)\n#     g_text = re.sub(r\"you'd\", \"You would\", g_text)\n#     g_text = re.sub(r\"It's\", \"It is\", g_text)\n#     g_text = re.sub(r\"Ain't\", \"am not\", g_text)\n#     g_text = re.sub(r\"Haven't\", \"Have not\", g_text)\n#     g_text = re.sub(r\"Could've\", \"Could have\", g_text)\n#     g_text = re.sub(r\"youve\", \"you have\", g_text)  \n#     g_text = re.sub(r\"donå«t\", \"do not\", g_text)   \n            \n#     # Character entity references\n#     g_text = re.sub(r\"&gt;\", \">\", g_text)\n#     g_text = re.sub(r\"&lt;\", \"<\", g_text)\n#     g_text = re.sub(r\"&amp;\", \"&\", g_text)\n    \n           \n#     # Urls\n#     g_text = re.sub(r\"https?:\\/\\/t.co\\/[A-Za-z0-9]+\", \"\", g_text)\n        \n#     # Words with punctuations and special characters\n#     punctuations = '@#!?+&*[]-%.:/();$=><|{}^' + \"'`\"\n#     for p in punctuations:\n#         g_text = g_text.replace(p, f' {p} ')\n        \n#     # ... and ..\n#     g_text = g_text.replace('...', ' ... ')\n#     if '...' not in g_text:\n#         g_text = g_text.replace('..', ' ... ')      \n    \n    \n#     g_text = re.sub(r\"\\'scuse\", \" excuse \", g_text)\n#     g_text = re.sub('\\W', ' ', g_text)\n#     g_text = re.sub('\\s+', ' ', g_text)\n#     g_text = g_text.strip(' ')\n#     return g_text","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df2['toxic'] = df2['toxic'].apply(lambda x: 0 if x<0.5 else 1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"df2 = pd.concat([\n    \n    df2[['comment_text', 'toxic']].query('toxic==1'),\n    df2[['comment_text', 'toxic']].query('toxic==0').sample(n=500000, random_state=0)\n])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Percentage of unlabelled comments\nunlabelled_df2_all = df2[(df2['toxic']!=1) ]\nprint('Percentage of unlabelled comments in df2 ', len(unlabelled_df2_all)/len(df2)*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# # clean the comment_text in df1\n# df1['comment_text'] = df1['comment_text'].map(lambda com : clean_text(com))\n# # clean the comment_text in df2\n# df2['comment_text'] = df2['comment_text'].map(lambda com : clean_text(com))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Character length for the rows in the df1 & df2 data\ndf1['char_length'] = df1['comment_text'].apply(lambda x: len(str(x)))\ndf2['char_length'] = df2['comment_text'].apply(lambda x: len(str(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Character length for the rows in the training data\ndf1= df1[df1['char_length'] >= 545] \ndf2= df2[df2['char_length'] >= 545] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train = pd.concat([df1, df2], axis=0).reset_index(drop=True)\ntrain = train.sample(frac=1).reset_index(drop=True).head(200000)\ntrain = train.reset_index(drop=True)\nvalid = df3\nvalid = valid.reset_index(drop=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# train['toxic'] = train['toxic'].apply(lambda x: 0 if x<0.5 else 1)","execution_count":null,"outputs":[]},{"metadata":{"_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","trusted":false},"cell_type":"code","source":"train.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Percentage of unlabelled comments\nunlabelled_in_all = train[(train['toxic']!=1) ]\nprint('Percentage of unlabelled comments is ', len(unlabelled_in_all)/len(train)*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# # check for any 'null' comment in training data\n# no_comment = train[train['comment_text'].isnull()]\n# len(no_comment)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# # check for any 'null' comment in test data\n# no_comment = test[test['content'].isnull()]\n# len(no_comment)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# # check for any 'null' comment in validation data\n# no_comment = valid[valid['comment_text'].isnull()]\n# len(no_comment)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# total rows in train, test valid data\nprint('Total rows in training dataset is {}'.format(len(train)))\nprint('Total rows in validation dataset is {}'.format(len(valid)))\nprint('Total rows in test dataset is {}'.format(len(test)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# # Character length for the rows in the training data\n# train['char_length'] = train['comment_text'].apply(lambda x: len(str(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# histogram plot for text length\nsns.set()\ntrain['char_length'].hist()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# top 10 largest values in column char_length \ntrain.nlargest(10, 'char_length') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# top 10 smallest values in column char_length \ntrain.nsmallest(10, 'char_length') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# # Character length for the rows in the training data\n# train= train[train['char_length'] >= 15] ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# # top 10 largest values in column char_length \n# train.nsmallest(10, 'char_length') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Percentage of unlabelled comments\nunlabelled_in_all = valid[(valid['toxic']!=1) ]\nprint('Percentage of unlabelled comments in validation is ', len(unlabelled_in_all)/len(valid)*100)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"valid.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"test['char_length'] = test['content'].apply(lambda x: len(str(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sns.set()\ntest['char_length'].hist()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# top 10 largest values test dataset  in column char_length \ntest.nlargest(10, 'char_length') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# top 10 smallest values test dataset  in column char_length \ntest.nsmallest(10, 'char_length') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"valid['char_length'] = valid['comment_text'].apply(lambda x: len(str(x)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sns.set()\nvalid['char_length'].hist()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# top 10 largest values valid dataset  in column char_length \nvalid.nlargest(10, 'char_length') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# top 10 smallest values valid dataset  in column char_length \nvalid.nsmallest(10, 'char_length') ","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train = train.drop('char_length',axis=1)\nvalid = valid.drop('char_length',axis=1)\ntest = test.drop('char_length',axis=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# # # Function to encode the text\ndef encode_fn(texts, tokenizer, maxlen=512):\n    encode = tokenizer.batch_encode_plus(\n        texts, \n        return_attention_masks=False, \n        return_token_type_ids=False,\n        pad_to_max_length=True,\n        max_length=maxlen\n    )\n    \n    return np.array(encode['input_ids'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n# Configuration\nEPOCHS = 3\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nMAX_LEN = 192","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# First load the real tokenizer\ntokenizer = transformers.AutoTokenizer.from_pretrained('jplu/tf-xlm-roberta-large')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\nx_train = encode_fn(train.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)\nx_valid = encode_fn(valid.comment_text.astype(str), tokenizer, maxlen=MAX_LEN)\nx_test =  encode_fn(test.content.astype(str), tokenizer, maxlen=MAX_LEN)\n\ny_train = train.toxic.values\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# Function to build the MODEL\ndef model_fn(transformer, max_len=512):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    \n    cls_token = sequence_output[:, 0, :]\n\n    \n    \n#     layers1 = Dense(512, kernel_regularizer=regularizers.l2(0.0001),activation='elu')(cls_token)\n  \n#     layers3 = Dense(512, kernel_regularizer=regularizers.l2(0.0001),activation='elu')(layers1)\n  \n#     layers5 = Dense(512, kernel_regularizer=regularizers.l2(0.0001),activation='elu')(layers3)\n  \n#     layers7 = Dense(512, kernel_regularizer=regularizers.l2(0.0001),activation='elu')(layers5)\n  \n    \n    \n    \n#     out = Dense(1, activation='sigmoid')(layers7)\n\n  \n    out = Dense(1, activation='sigmoid')(cls_token)\n    model = Model(inputs=input_word_ids, outputs=out)\n    optimizer = tf.keras.optimizers.Adam(\n    learning_rate=0.00001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False,\n    name='Adam'\n)\n    model.compile(optimizer, loss='binary_crossentropy', metrics = [tf.keras.metrics.AUC()] )\n    return model\n    \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    transformer_layer = (\n        transformers.TFAutoModel.from_pretrained(\"jplu/tf-xlm-roberta-large\")   \n        \n    )\n    model = model_fn(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# n_steps = x_train.shape[0] // BATCH_SIZE\n# train_history = model.fit(\n#     train_dataset,\n#     steps_per_epoch=n_steps,\n#     #callbacks=[callback],\n#     validation_data=valid_dataset,\n#     epochs=EPOCHS\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# n_steps = x_valid.shape[0] // BATCH_SIZE\n# train_history_2 = model.fit(\n#     valid_dataset.repeat(),\n#     steps_per_epoch=n_steps,\n#     epochs=EPOCHS*2\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# sample['toxic'] = model.predict(test_dataset, verbose=1)\n# sample.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"# from keras.callbacks import EarlyStopping, ModelCheckpoint, LearningRateScheduler\n\n# file_path = \"best_model.hdf5\"\n# check_point = ModelCheckpoint(file_path, monitor = \"val_loss\", verbose = 1,\n#                               save_best_only = True, mode = \"min\")\n# #ra_val = RocAucEvaluation(validation_data=valid_dataset, interval = 1)\n# early_stop = EarlyStopping(monitor = \"val_loss\", mode = \"min\", patience = 5)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"\n# n_steps = x_train.shape[0] // BATCH_SIZE\n\n# train_history = model.fit(\n#     train_dataset,\n#     steps_per_epoch=n_steps,\n#     validation_steps=127,\n#     #callbacks = [check_point, early_stop],\n#     validation_data=valid_dataset,\n#     epochs=4\n# )","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"callback_stop = tf.keras.callbacks.EarlyStopping(\n    monitor='val_loss', min_delta=0, patience=5, verbose=0, mode='auto',\n    baseline=None, restore_best_weights=True\n)\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"n_steps = x_train.shape[0]// BATCH_SIZE\ntrain_history1 = model.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    callbacks = [callback_stop],\n    validation_data=valid_dataset,\n    epochs=20\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"n_steps = x_valid.shape[0] // BATCH_SIZE\ntrain_history_2 = model.fit(\n    valid_dataset.repeat(),\n    steps_per_epoch=n_steps,\n    callbacks = [callback_stop],\n    epochs=EPOCHS*2\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":false},"cell_type":"code","source":"sample['toxic'] = model.predict(test_dataset, verbose=1)\nsample.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}