{"cells":[{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"!pip install -q pyicu\n!pip install -q pycld2\n!pip install -q polyglot\n!pip install -q textstat\n!pip install -q googletrans","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true,"_kg_hide-input":true,"_kg_hide-output":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport gc\nimport re\nimport folium\nimport textstat\nfrom scipy import stats\nfrom colorama import Fore, Back, Style, init\n\nimport math\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\n\nimport random\nimport networkx as nx\nfrom pandas import Timestamp\n\nfrom PIL import Image\nfrom IPython.display import SVG\nfrom keras.utils import model_to_dot\n\nimport requests\nfrom IPython.display import HTML\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\n\ntqdm.pandas()\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\nimport transformers\nimport tensorflow as tf\n\nfrom tensorflow.keras.callbacks import Callback\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\n\nfrom tensorflow.keras.models import Model\nfrom kaggle_datasets import KaggleDatasets\nfrom tensorflow.keras.optimizers import Adam\nfrom tokenizers import BertWordPieceTokenizer\nfrom tensorflow.keras.layers import Dense, Input, Dropout, Embedding\nfrom tensorflow.keras.layers import LSTM, GRU, Conv1D, SpatialDropout1D\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import activations\nfrom tensorflow.keras import constraints\nfrom tensorflow.keras import initializers\nfrom tensorflow.keras import regularizers\n\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.activations import *\nfrom tensorflow.keras.constraints import *\nfrom tensorflow.keras.initializers import *\nfrom tensorflow.keras.regularizers import *\n\nfrom sklearn import metrics\nfrom sklearn.utils import shuffle\nfrom gensim.models import Word2Vec\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_extraction.text import TfidfVectorizer,\\\n                                            CountVectorizer,\\\n                                            HashingVectorizer\n\nfrom nltk.stem.wordnet import WordNetLemmatizer \nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer  \n\nimport nltk\nfrom textblob import TextBlob\n\nfrom nltk.corpus import wordnet\nfrom nltk.corpus import stopwords\nfrom googletrans import Translator\nfrom nltk import WordNetLemmatizer\nfrom polyglot.detect import Detector\nfrom nltk.stem import WordNetLemmatizer\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\n\nstopword=set(STOPWORDS)\n\nlem = WordNetLemmatizer()\ntokenizer=TweetTokenizer()\n\nnp.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load the training, validation, and testing datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"DATA_PATH = \"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/\"\nos.listdir(DATA_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"TEST_PATH = DATA_PATH + \"test.csv\"\nVAL_PATH = DATA_PATH + \"validation.csv\"\nTRAIN_PATH = DATA_PATH + \"jigsaw-toxic-comment-train.csv\"\n\nval_data = pd.read_csv(VAL_PATH)\ntest_data = pd.read_csv(TEST_PATH)\ntrain_data = pd.read_csv(TRAIN_PATH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val = val_data\ntrain = train_data\n\ndef clean(text):\n    text = text.fillna(\"fillna\").str.lower()\n    text = text.map(lambda x: re.sub('\\\\n',' ',str(x)))\n    text = text.map(lambda x: re.sub(\"\\[\\[User.*\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n    return text\n\nval[\"comment_text\"] = clean(val[\"comment_text\"])\ntest_data[\"content\"] = clean(test_data[\"content\"])\ntrain[\"comment_text\"] = clean(train[\"comment_text\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RocAucEvaluation(Callback):\n    def __init__(self, validation_data=(), interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.X_val, self.y_val = validation_data\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict(self.X_val, verbose=0)\n            score = roc_auc_score(self.y_val, y_pred)\n            print(\"\\n ROC-AUC - epoch: {:d} - score: {:.6f}\".format(epoch+1, score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=240, maxlen=512):\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in range(0, len(texts), chunk_size):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Setup TPU configuration","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\ntpu = tf.distribute.cluster_resolver.TPUClusterResolver()\ntf.config.experimental_connect_to_cluster(tpu)\ntf.tpu.experimental.initialize_tpu_system(tpu)\nstrategy = tf.distribute.experimental.TPUStrategy(tpu)\n\nGCS_DS_PATH = KaggleDatasets().get_gcs_path('jigsaw-multilingual-toxic-comment-classification')\n\nEPOCHS = 2\nBATCH_SIZE = 32 * strategy.num_replicas_in_sync","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Load BERT tokenizer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\n\nsave_path = '/kaggle/working/distilbert_base_uncased/'\nif not os.path.exists(save_path):\n    os.makedirs(save_path)\ntokenizer.save_pretrained(save_path)\n\nfast_tokenizer = BertWordPieceTokenizer('distilbert_base_uncased/vocab.txt', \n                                        lowercase=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Encode comments and get targets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train = fast_encode(train.comment_text.astype(str), \n                      fast_tokenizer, maxlen=512)\nx_valid = fast_encode(val_data.comment_text.astype(str).values, \n                      fast_tokenizer, maxlen=512)\nx_test = fast_encode(test_data.content.astype(str).values, \n                     fast_tokenizer, maxlen=512)\n\ny_valid = val.toxic.values\ny_train = train.toxic.values","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define training, validation, and testing datasets","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Vanilla neural network <a id=\"2.2\"></a>\n\nVanilla neural network refers to the classic neural network architecture.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"The same logic is implemented below using tf.keras.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Define VNN model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_vnn_model(transformer, max_len):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    \n    embed = transformer.weights[0].numpy()\n    embedding = Embedding(np.shape(embed)[0], np.shape(embed)[1],\n                          input_length=max_len, weights=[embed],\n                          trainable=False)(input_word_ids)\n    \n    conc = K.sum(embedding, axis=2)\n    conc = Dense(128, activation='relu')(conc)\n    conc = Dense(1, activation='sigmoid')(conc)\n    \n    model = Model(inputs=input_word_ids, outputs=conc)\n    \n    model.compile(Adam(lr=0.01), \n                  loss='binary_crossentropy', \n                  metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Build model and check summary","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"with strategy.scope():\n    transformer_layer = transformers.TFDistilBertModel.\\\n    from_pretrained('distilbert-base-multilingual-cased')\n    model_vnn = build_vnn_model(transformer_layer, max_len=512)\n\nmodel_vnn.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize network architecture","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"SVG(tf.keras.utils.model_to_dot(model_vnn, dpi=70).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define ReduceLROnPlateau callback","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def callback():\n    cb = []\n\n    reduceLROnPlat = ReduceLROnPlateau(monitor='val_loss',  \n                                    factor=0.3, patience=3, \n                                    verbose=1, mode='auto', \n                                    epsilon=0.0001, cooldown=1, min_lr=0.000001)\n    cb.append(reduceLROnPlat)\n    log = CSVLogger('log.csv')\n    cb.append(log)\n\n    RocAuc = RocAucEvaluation(validation_data=(x_valid, y_valid), interval=1)\n    cb.append(RocAuc)\n    \n    return cb","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"N_STEPS = x_train.shape[0] // BATCH_SIZE\ncalls = callback()\n\ntrain_history = model_vnn.fit(\n    train_dataset,\n    steps_per_epoch=N_STEPS,\n    validation_data=valid_dataset,\n    callbacks = calls,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize model predictions\n\nNow, I will visualize the performance of the model on few validation samples.","execution_count":null},{"metadata":{"trusted":true,"_kg_hide-input":true},"cell_type":"code","source":"translator = Translator()\n\ndef visualize_model_preds(model, indices=[0, 17, 1, 24]):\n    comments = val_data.comment_text.loc[indices].values.tolist()\n    preds = model.predict(x_valid[indices].reshape(len(indices), -1))\n\n    for idx, i in enumerate(indices):\n        if y_valid[i] == 0:\n            label = \"Non-toxic\"\n            color = f'{Fore.GREEN}'\n            symbol = '\\u2714'\n        else:\n            label = \"Toxic\"\n            color = f'{Fore.RED}'\n            symbol = '\\u2716'\n\n        print('{}{} {}'.format(color, str(idx+1) + \". \" + label, symbol))\n        print(f'{Style.RESET_ALL}')\n        print(\"ORIGINAL\")\n        print(comments[idx]); print(\"\")\n        print(\"TRANSLATED\")\n        print(translator.translate(comments[idx]).text)\n        fig = go.Figure()\n        if list.index(sorted(preds[:, 0]), preds[idx][0]) > 1:\n            yl = [preds[idx][0], 1 - preds[idx][0]]\n        else:\n            yl = [1 - preds[idx][0], preds[idx][0]]\n        fig.add_trace(go.Bar(x=['Non-Toxic', 'Toxic'], y=yl, marker=dict(color=[\"seagreen\", \"indianred\"])))\n        fig.update_traces(name=comments[idx])\n        fig.update_layout(xaxis_title=\"Labels\", yaxis_title=\"Probability\", template=\"plotly_white\", title_text=\"Predictions for validation comment #{}\".format(idx+1))\n        fig.show()\n        \nvisualize_model_preds(model_vnn)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can see that the model gets only two out of four answers correct. We need to find a better model to achieve a higher accuracy.","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"## LSTM with Attention <a id=\"2.4\"></a>\n ","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"### Define the Attention layer","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"class AttentionWeightedAverage(Layer):\n\n    def __init__(self, return_attention=False, **kwargs):\n        self.init = initializers.get('uniform')\n        self.supports_masking = True\n        self.return_attention = return_attention\n        super(AttentionWeightedAverage, self).__init__(** kwargs)\n\n    def build(self, input_shape):\n        self.input_spec = [InputSpec(ndim=3)]\n        assert len(input_shape) == 3\n\n        self.W = self.add_weight(shape=(input_shape[2], 1),\n                                 name='{}_W'.format(self.name),\n                                 initializer=self.init)\n        super(AttentionWeightedAverage, self).build(input_shape)\n\n    def call(self, x, mask=None):\n        logits = K.dot(x, self.W)\n        x_shape = K.shape(x)\n        logits = K.reshape(logits, (x_shape[0], x_shape[1]))\n        ai = K.exp(logits - K.max(logits, axis=-1, keepdims=True))\n\n        if mask is not None:\n            mask = K.cast(mask, K.floatx())\n            ai = ai * mask\n        att_weights = ai / (K.sum(ai, axis=1, keepdims=True) + K.epsilon())\n        weighted_input = x * K.expand_dims(att_weights)\n        result = K.sum(weighted_input, axis=1)\n        if self.return_attention:\n            return [result, att_weights]\n        return result\n\n    def get_output_shape_for(self, input_shape):\n        return self.compute_output_shape(input_shape)\n\n    def compute_output_shape(self, input_shape):\n        output_len = input_shape[2]\n        if self.return_attention:\n            return [(input_shape[0], output_len), (input_shape[0], input_shape[1])]\n        return (input_shape[0], output_len)\n\n    def compute_mask(self, input, input_mask=None):\n        if isinstance(input_mask, list):\n            return [None] * len(input_mask)\n        else:\n            return None","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Define the LSTM model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def build_lstm_model(transformer, max_len):\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    \n    embed = transformer.weights[0].numpy()\n    embedding = Embedding(np.shape(embed)[0], np.shape(embed)[1],\n                          input_length=max_len, weights=[embed],\n                          trainable=False)(input_word_ids)\n    \n    embedding = SpatialDropout1D(0.3)(embedding)\n    lstm_1 = LSTM(512, return_sequences=True)(embedding)\n    lstm_2 = LSTM(512, return_sequences=True)(lstm_1)\n    \n    attention = AttentionWeightedAverage()(lstm_2)\n    conc = Dense(64, activation='relu')(attention)\n    conc = Dense(1, activation='sigmoid')(conc)\n    \n    model = Model(inputs=input_word_ids, outputs=conc)\n    \n    model.compile(Adam(lr=0.01), \n                  loss='binary_crossentropy', \n                  metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Build the model and check summary","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    model_lstm = build_lstm_model(transformer_layer, max_len=512)\n\nmodel_lstm.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize network architecture","execution_count":null},{"metadata":{"_kg_hide-input":true,"trusted":true},"cell_type":"code","source":"SVG(tf.keras.utils.model_to_dot(model_lstm, dpi=70).create(prog='dot', format='svg'))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Train the model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_history = model_lstm.fit(\n    train_dataset,\n    steps_per_epoch=N_STEPS,\n    validation_data=valid_dataset,\n    callbacks = calls,\n    epochs=EPOCHS\n)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Visualize model predictions\n\n**Note: There are some errors which I am trying to fix here. The predictions cannot be visualized correctly.** But, I expect that LSTMs would perform well as they can understand text data well with attention.","execution_count":null},{"metadata":{"_kg_hide-input":true,"_kg_hide-output":true,"trusted":true},"cell_type":"code","source":"visualize_model_preds(model_lstm)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Generate submission","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(DATA_PATH + 'sample_submission.csv')\nvnn_pred = model_vnn.predict(test_dataset, verbose=1)\n# sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.read_csv(DATA_PATH + 'sample_submission.csv')\nlstm_pred = model_lstm.predict(test_dataset, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub[\"toxic\"] = lstm_pred[:-4]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}