{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Dependencies","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import os\nimport gc\n\nimport numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\nimport tensorflow as tf\n\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.models import Model, Sequential\nfrom tensorflow.keras.layers import Dense, Flatten, Dropout, Embedding, SimpleRNN, Input\nfrom tensorflow.keras.optimizers import Adam\nfrom tensorflow.keras.callbacks import ModelCheckpoint\n\nfrom transformers import XLMRobertaTokenizer, TFXLMRobertaModel\nfrom tqdm.notebook import tqdm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Loading Dataset","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train1_csv = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain1 = train1_csv[['comment_text', 'toxic']]\ndisplay(train1.head())\nprint(train1.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train1.toxic.value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train2_csv = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n# train2_high = train2_csv[['comment_text', 'toxic']][train2_csv.toxic > 0.80].reset_index(drop=True)\n# train2_low = train2_csv[['comment_text', 'toxic']][train2_csv.toxic == 0.00]\\\n#             .sample(frac=0.05, replace=False, random_state=42).reset_index(drop=True)\n# display(train2_high.head())\n# print(\"toxic > 0.75\", train2_high.shape)\n# display(train2_low.head())\n# print(\"toxic = 0.00\", train2_low.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train = pd.concat([train1, train2_high, train2_low], ignore_index=True)\n# display(train.head())\n# print(train.shape)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv\")\nvalid = valid[['comment_text', 'lang', 'toxic']]\ndisplay(valid.head())\nvalid.lang.value_counts(normalize=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Combine training data with non-english samples from valid\nvalid_samples = valid.groupby('lang', group_keys=False)\\\n                     .apply(lambda g: g.sample(frac=0.5, random_state=1))\\\n                     .reset_index(drop=True)\nvalid_samples","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.concat([train1, valid_samples], ignore_index=True)\ntrain","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cond = valid.comment_text.isin(valid_samples.comment_text)\nvalid.drop(valid[cond].index, inplace = True)\nvalid","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Preprocessing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = Tokenizer(oov_token=\"<OOV>\")\ntokenizer.fit_on_texts(train.comment_text.values)\nsequences = tokenizer.texts_to_sequences(train.comment_text.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"reverse_word_map = dict(map(reversed, tokenizer.word_index.items()))\n\ndef sequence_to_text(list_of_indices):\n    # Looking up words in dictionary\n    words = [reverse_word_map.get(letter) for letter in list_of_indices]\n    return(words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.comment_text.iloc[213924]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sequence_to_text(sequences[213924])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_sequences = tokenizer.texts_to_sequences(valid.comment_text.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded = pad_sequences(sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)\nvalid_padded = pad_sequences(valid_sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded[0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sequence_to_text(train_padded[0])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Simple NN - Embeddings Only","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# output directory\nOUTPUT_DIR = '../input/output/dense'\n\n# training\nEPOCHS = 4\nBATCH_SIZE = 128\n\n# embedding\nN_DIM = 64\nN_UNIQUE_WORDS = 5000\nN_WORDS_TO_SKIP = 50\nMAX_LENGTH = 100\nPAD_TYPE = TRUNC_TYPE = 'pre'\n\n# NN architecture\nN_DENSE = 64\nDROPOUT = 0.5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = Sequential()\n    model.add(Embedding(N_UNIQUE_WORDS, N_DIM, input_length=MAX_LENGTH))\n    model.add(Flatten())\n    model.add(Dense(N_DENSE, activation='relu'))\n    model.add(Dropout(DROPOUT))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"gc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"modelcheckpoint = ModelCheckpoint(\"/kaggle/working/dense/best.hdf5\", \\\n                                  monitor='valloss', savebest_only=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_padded, train.toxic.values, \\\n          batch_size = BATCH_SIZE, epochs = EPOCHS, verbose=2, \\\n          validation_data=(valid_padded, valid.toxic.values), \\\n          callbacks=[modelcheckpoint])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_hat = model.predict_proba(valid_padded)\nplt.hist(y_hat)\n_ = plt.axvline(x=0.5, color='orange')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model with CNN","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import SpatialDropout1D, Conv1D, GlobalMaxPooling1D","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# training:\nEPOCHS = 4\nBATCH_SIZE = 128\n\n# vector-space embedding: \nN_DIM = 64\nN_UNIQUE_WORDS = 20000 \nMAX_LEN = 400\nPAD_TYPE = TRUNC_TYPE = 'pre'\nDROP_EMBED = 0.2 \n\n# convolutional layer architecture:\nN_CONV = 256 \nK_CONV = 3 \n\n# dense layer architecture: \nN_DENSE = 256\nDROPOUT = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded = pad_sequences(sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)\nvalid_padded = pad_sequences(valid_sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = Sequential()\n    model.add(Embedding(N_UNIQUE_WORDS, N_DIM, input_length=MAX_LENGTH))\n    model.add(SpatialDropout1D(DROP_EMBED))\n    model.add(Conv1D(N_CONV, K_CONV, activation='relu'))\n    model.add(GlobalMaxPooling1D())\n    model.add(Dense(N_DENSE, activation='relu'))\n    model.add(Dropout(DROPOUT))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_padded, train.toxic.values, \\\n          batch_size = BATCH_SIZE, epochs = EPOCHS, verbose=2, \\\n          validation_data=(valid_padded, valid.toxic.values), \\\n          callbacks=[modelcheckpoint])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model with Vanila RNN","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# training:\nEPOCHS = 16\nBATCH_SIZE = 128\n\n# vector-space embedding: \nN_DIM = 64\nN_UNIQUE_WORDS = 20000 \nMAX_LEN = 100\nPAD_TYPE = TRUNC_TYPE = 'pre'\nDROP_EMBED = 0.2 \n\n# convolutional layer architecture:\nN_RNN = 256 \nDROP_RNN = 0.2\n\n# dense layer architecture: \nN_DENSE = 256\nDROPOUT = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded = pad_sequences(sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)\nvalid_padded = pad_sequences(valid_sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = Sequential()\n    model.add(Embedding(N_UNIQUE_WORDS, N_DIM, input_length=MAX_LENGTH))\n    model.add(SpatialDropout1D(DROP_EMBED))\n    model.add(SimpleRNN(N_RNN, activation='relu'))\n    model.add(Dense(N_DENSE, activation='relu'))\n    model.add(Dropout(DROPOUT))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_padded, train.toxic.values, \\\n          batch_size = BATCH_SIZE, epochs = EPOCHS, verbose=2, \\\n          validation_data=(valid_padded, valid.toxic.values), \\\n          callbacks=[modelcheckpoint])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model with GRU (Gated Recurrent Unit)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import GRU","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# training:\nEPOCHS = 4\nBATCH_SIZE = 128\n\n# vector-space embedding: \nN_DIM = 64\nN_UNIQUE_WORDS = 20000 \nMAX_LEN = 100\nPAD_TYPE = TRUNC_TYPE = 'pre'\nDROP_EMBED = 0.2 \n\n# convolutional layer architecture:\nN_GRU = 256 \nDROP_GRU = 0.2\n\n# dense layer architecture: \nN_DENSE = 256\nDROPOUT = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded = pad_sequences(sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)\nvalid_padded = pad_sequences(valid_sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = Sequential()\n    model.add(Embedding(N_UNIQUE_WORDS, N_DIM, input_length=MAX_LENGTH))\n    model.add(SpatialDropout1D(DROP_EMBED))\n    model.add(GRU(N_GRU, dropout=DROP_GRU))\n    model.add(Dense(N_DENSE, activation='relu'))\n    model.add(Dropout(DROPOUT))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_padded, train.toxic.values, \\\n          batch_size = BATCH_SIZE, epochs = EPOCHS, verbose=2, \\\n          validation_data=(valid_padded, valid.toxic.values), \\\n          callbacks=[modelcheckpoint])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model with LSTM (Long Short Term Memory unit)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import LSTM","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# training:\nEPOCHS = 4\nBATCH_SIZE = 128\n\n# vector-space embedding: \nN_DIM = 64\nN_UNIQUE_WORDS = 20000 \nMAX_LEN = 100\nPAD_TYPE = TRUNC_TYPE = 'pre'\nDROP_EMBED = 0.2 \n\n# convolutional layer architecture:\nN_LSTM = 256 \nDROP_LSTM = 0.2\n\n# dense layer architecture: \nN_DENSE = 256\nDROPOUT = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded = pad_sequences(sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)\nvalid_padded = pad_sequences(valid_sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = Sequential()\n    model.add(Embedding(N_UNIQUE_WORDS, N_DIM, input_length=MAX_LENGTH))\n    model.add(SpatialDropout1D(DROP_EMBED))\n    model.add(LSTM(N_LSTM, dropout=DROP_LSTM))\n    model.add(Dense(N_DENSE, activation='relu'))\n    model.add(Dropout(DROPOUT))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_padded, train.toxic.values, \\\n          batch_size = BATCH_SIZE, epochs = EPOCHS, verbose=2, \\\n          validation_data=(valid_padded, valid.toxic.values), \\\n          callbacks=[modelcheckpoint])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model with Bi-LSTM","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import Bidirectional","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# training:\nEPOCHS = 6\nBATCH_SIZE = 128\n\n# vector-space embedding: \nN_DIM = 64\nN_UNIQUE_WORDS = 20000 \nMAX_LEN = 200\nPAD_TYPE = TRUNC_TYPE = 'pre'\nDROP_EMBED = 0.2 \n\n# convolutional layer architecture:\nN_LSTM = 256 \nDROP_LSTM = 0.2\n\n# dense layer architecture: \nN_DENSE = 256\nDROPOUT = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded = pad_sequences(sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)\nvalid_padded = pad_sequences(valid_sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = Sequential()\n    model.add(Embedding(N_UNIQUE_WORDS, N_DIM, input_length=MAX_LENGTH))\n    model.add(SpatialDropout1D(DROP_EMBED))\n    model.add(Bidirectional(LSTM(N_LSTM, dropout=DROP_LSTM)))\n    model.add(Dense(N_DENSE, activation='relu'))\n    model.add(Dropout(DROPOUT))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_padded, train.toxic.values, \\\n          batch_size = BATCH_SIZE, epochs = EPOCHS, verbose=2, \\\n          validation_data=(valid_padded, valid.toxic.values), \\\n          callbacks=[modelcheckpoint])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model with Stacked Bi-LSTM","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# training:\nEPOCHS = 4\nBATCH_SIZE = 128\n\n# vector-space embedding: \nN_DIM = 64\nN_UNIQUE_WORDS = 20000 \nMAX_LEN = 200\nPAD_TYPE = TRUNC_TYPE = 'pre'\nDROP_EMBED = 0.2 \n\n# convolutional layer architecture:\nN_LSTM = 256\nDROP_LSTM = 0.2\n\n# dense layer architecture: \nN_DENSE = 256\nDROPOUT = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded = pad_sequences(sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)\nvalid_padded = pad_sequences(valid_sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = Sequential()\n    model.add(Embedding(N_UNIQUE_WORDS, N_DIM, input_length=MAX_LENGTH))\n    model.add(SpatialDropout1D(DROP_EMBED))\n    model.add(Bidirectional(LSTM(N_LSTM, dropout=DROP_LSTM, return_sequences=True)))\n    model.add(Bidirectional(LSTM(N_LSTM, dropout=DROP_LSTM)))\n    model.add(Dense(N_DENSE, activation='relu'))\n    model.add(Dropout(DROPOUT))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_padded, train.toxic.values, \\\n          batch_size = BATCH_SIZE, epochs = EPOCHS, verbose=2, \\\n          validation_data=(valid_padded, valid.toxic.values), \\\n          callbacks=[modelcheckpoint])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Model with Conv, LSTM Stack","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.layers import MaxPooling1D","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# training:\nEPOCHS = 4\nBATCH_SIZE = 128\n\n# vector-space embedding: \nN_DIM = 64\nN_UNIQUE_WORDS = 40000 \nMAX_LEN = 200\nPAD_TYPE = TRUNC_TYPE = 'pre'\nDROP_EMBED = 0.2 \n\n# convolutional layer architecture:\nN_CONV = 64\nK_CONV = 3\nMP_SIZE = 4\n\n# LSTM layer architecture:\nN_LSTM = 64\nDROP_LSTM = 0.2\n\n# dense layer architecture: \nN_DENSE = 256\nDROPOUT = 0.2","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_padded = pad_sequences(sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)\nvalid_padded = pad_sequences(valid_sequences, maxlen=MAX_LENGTH, padding=PAD_TYPE, \\\n                             truncating=TRUNC_TYPE, value=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = Sequential()\n    model.add(Embedding(N_UNIQUE_WORDS, N_DIM, input_length=MAX_LENGTH))\n    model.add(SpatialDropout1D(DROP_EMBED))\n    model.add(Conv1D(N_CONV, K_CONV, activation='relu'))\n    model.add(MaxPooling1D(MP_SIZE))\n    model.add(Bidirectional(LSTM(N_LSTM, dropout=DROP_LSTM)))\n    model.add(Dense(N_DENSE, activation='relu'))\n    model.add(Dropout(DROPOUT))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(train_padded, train.toxic.values, \\\n          batch_size = BATCH_SIZE, epochs = EPOCHS, verbose=2, \\\n          validation_data=(valid_padded, valid.toxic.values), \\\n          callbacks=[modelcheckpoint])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"\n\n## Using Pretrained Embeddings - GloVe","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!wget http://nlp.stanford.edu/data/glove.6B.zip\n!unzip -q glove.6B.zip","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"path_to_glove_file = \"../input/glove-global-vectors-for-word-representation/glove.6B.100d.txt\"\n\nembeddings_index = {}\nwith open(path_to_glove_file) as f:\n    for line in f:\n        word, coefs = line.split(maxsplit=1)\n        coefs = np.fromstring(coefs, \"f\", sep=\" \")\n        embeddings_index[word] = coefs\n\nprint(\"Found %s word vectors.\" % len(embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_index = tokenizer.word_index\nnum_tokens = len(word_index) + 1\nembedding_dim = 100\nhits = 0\nmisses = 0\n\n# Prepare embedding matrix\nembedding_matrix = np.zeros((num_tokens, embedding_dim))\nfor word, i in word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        # Words not found in embedding index will be all-zeros.\n        # This includes the representation for \"padding\" and \"OOV\"\n        embedding_matrix[i] = embedding_vector\n        hits += 1\n    else:\n        misses += 1\nprint(\"Converted %d words (%d misses)\" % (hits, misses))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from keras import initializers\n\nwith strategy.scope():\n    model = tf.keras.Sequential([\n            tf.keras.layers.Embedding(input_dim=embedding_matrix.shape[0], output_dim=100, \\\n                                      embeddings_initializer=initializers.Constant(embedding_matrix), \\\n                                      trainable=False),\n            tf.keras.layers.GlobalAveragePooling1D(),\n            tf.keras.layers.Dense(6, activation='relu'),\n            tf.keras.layers.Dense(1, activation='sigmoid')])\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(padded, train1.toxic.values, epochs = 10, validation_data=(valid_padded, valid.toxic.values), verbose=2)\n#0.5760","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model = tf.keras.Sequential([\n            tf.keras.layers.Embedding(input_dim=embedding_matrix.shape[0], output_dim=100, \\\n                                      embeddings_initializer=initializers.Constant(embedding_matrix), \\\n                                      trainable=False),\n            tf.keras.layers.LSTM(32),\n            tf.keras.layers.Dense(6, activation='relu'),\n            tf.keras.layers.Dense(1, activation='sigmoid')])\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(padded, train1.toxic.values, epochs = 5, validation_data=(valid_padded, valid.toxic.values), verbose=2)\n# 0.8461","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"## Processing Inputs (Tokenizing X)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = XLMRobertaTokenizer.from_pretrained('xlm-roberta-large')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def regular_encode(texts, tokenizer, maxlen=512):\n    enc_di = []\n    for text in texts:\n        enc = tokenizer.encode(texts, \n                               return_attention_masks=False, \n                               return_token_type_ids=False,\n                               pad_to_max_length=True,\n                               max_length=maxlen)\n        enc_di.append(enc)\n    \n    return np.array(enc_di)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer.encode(train.comment_text[0], \n                               return_attention_masks=False, \n                               return_token_type_ids=False,\n                               pad_to_max_length=True,\n                               max_length=maxlen)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"texts = train.comment_text\nfor text in texts:\n    print(text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"[x for x in regular_encode(train.comment_text[0], tokenizer, 512)]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = regular_encode(train.comment_text.values, tokenizer, 512)\nx_valid = regular_encode(valid.comment_text.values, tokenizer, 512)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train = train.toxic.values\ny_valid = valid.toxic.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Building Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"del train1_csv, train2_csv, train1, train2\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"del train2_high, train2_low\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prompt tf.data runtime to tune the prefetch value dynamically at runtime.\nAUTO = tf.data.experimental.AUTOTUNE\n\nEPOCHS = 3\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (tf.data.Dataset\\\n                .from_tensor_slices((x_train, y_train))\\\n                .shuffle(2048)\\\n                .batch(BATCH_SIZE)\\\n                .prefetch(AUTO))\n\nvalid_dataset = (tf.data.Dataset\\\n                .from_tensor_slices((x_valid, y_valid))\\\n                .batch(BATCH_SIZE)\\\n                .cache()\\\n                .prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_model(transformer, max_len=512):\n    inp = Input(shape=(max_len, ), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(inp)[0]\n    cls_token = sequence_output[:, 0, :]\n    out = Dense(1, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=inp, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    transformer_layer = TFXLMRobertaModel.from_pretrained('jplu/tf-xlm-roberta-large')\n    model_f = build_model(transformer_layer, max_len=MAX_LEN)\nmodel_f.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_train.shape[0] // BATCH_SIZE\ntrain_history = model_f.fit(\n    train_dataset,\n    steps_per_epoch=n_steps,\n    validation_data=valid_dataset,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Prediction","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv\")\ntest.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nsub.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_test = regular_encode(test.content.values, tokenizer)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset = (tf.data.Dataset\\\n                .from_tensor_slices(x_test)\\\n                .batch(BATCH_SIZE))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub['toxic'] = model_f.predict(test_dataset, verbose=1)\nsub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}