{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras import layers\nimport glob\nfrom sklearn import model_selection \nfrom tensorflow.keras.layers.experimental.preprocessing import TextVectorization\nimport string\nimport re\nfrom keras.preprocessing.text import Tokenizer\nfrom numpy import random\nfrom random import randint\nimport matplotlib.pyplot as plt\n#from tensorflow.python.keras.utils.data_utils import Sequence\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\n\n\nclass MultiHeadSelfAttention(layers.Layer):\n    def __init__(self, embed_dim, num_heads=8):\n        super(MultiHeadSelfAttention, self).__init__()\n        self.embed_dim = embed_dim\n        self.num_heads = num_heads\n        if embed_dim % num_heads != 0:\n            raise ValueError(\n                f\"embedding dimension = {embed_dim} should be divisible by number of heads = {num_heads}\"\n            )\n        self.projection_dim = embed_dim // num_heads\n        self.query_dense = layers.Dense(embed_dim)\n        self.key_dense = layers.Dense(embed_dim)\n        self.value_dense = layers.Dense(embed_dim)\n        self.combine_heads = layers.Dense(embed_dim)\n\n    def attention(self, query, key, value):\n        score = tf.matmul(query, key, transpose_b=True)\n        dim_key = tf.cast(tf.shape(key)[-1], tf.float32)\n        scaled_score = score / tf.math.sqrt(dim_key)\n        weights = tf.nn.softmax(scaled_score, axis=-1)\n        output = tf.matmul(weights, value)\n        return output, weights\n\n    def separate_heads(self, x, batch_size):\n        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.projection_dim))\n        return tf.transpose(x, perm=[0, 2, 1, 3])\n\n    def call(self, inputs):\n        # x.shape = [batch_size, seq_len, embedding_dim]\n        batch_size = tf.shape(inputs)[0]\n        query = self.query_dense(inputs)  # (batch_size, seq_len, embed_dim)\n        key = self.key_dense(inputs)  # (batch_size, seq_len, embed_dim)\n        value = self.value_dense(inputs)  # (batch_size, seq_len, embed_dim)\n        query = self.separate_heads(\n            query, batch_size\n        )  # (batch_size, num_heads, seq_len, projection_dim)\n        key = self.separate_heads(\n            key, batch_size\n        )  # (batch_size, num_heads, seq_len, projection_dim)\n        value = self.separate_heads(\n            value, batch_size\n        )  # (batch_size, num_heads, seq_len, projection_dim)\n        attention, weights = self.attention(query, key, value)\n        attention = tf.transpose(\n            attention, perm=[0, 2, 1, 3]\n        )  # (batch_size, seq_len, num_heads, projection_dim)\n        concat_attention = tf.reshape(\n            attention, (batch_size, -1, self.embed_dim)\n        )  # (batch_size, seq_len, embed_dim)\n        output = self.combine_heads(\n            concat_attention\n        )  # (batch_size, seq_len, embed_dim)\n        return output\n\n\n    \nclass TransformerBlock(layers.Layer):\n    def __init__(self, embed_dim, num_heads, ff_dim, rate=0.1):\n        super(TransformerBlock, self).__init__()\n        self.att = MultiHeadSelfAttention(embed_dim, num_heads)\n        self.ffn = keras.Sequential(\n            [layers.Dense(ff_dim, activation=\"relu\"), layers.Dense(embed_dim),]\n        )\n        self.layernorm1 = layers.LayerNormalization(epsilon=1e-6)\n        self.layernorm2 = layers.LayerNormalization(epsilon=1e-6)\n        self.dropout1 = layers.Dropout(rate)\n        self.dropout2 = layers.Dropout(rate)\n\n    def call(self, inputs, training):\n        attn_output = self.att(inputs)\n        attn_output = self.dropout1(attn_output, training=training)\n        out1 = self.layernorm1(inputs + attn_output)\n        ffn_output = self.ffn(out1)\n        ffn_output = self.dropout2(ffn_output, training=training)\n        return self.layernorm2(out1 + ffn_output)\n\n\n    \nclass TokenAndPositionEmbedding(layers.Layer):\n    def __init__(self, maxlen, vocab_size, embed_dim):\n        super(TokenAndPositionEmbedding, self).__init__()\n        self.token_emb = layers.Embedding(input_dim=vocab_size, output_dim=embed_dim)\n        self.pos_emb = layers.Embedding(input_dim=maxlen, output_dim=embed_dim)\n\n    def call(self, x):\n        maxlen = tf.shape(x)[-1]\n        positions = tf.range(start=0, limit=maxlen, delta=1)\n        positions = self.pos_emb(positions)\n        x = self.token_emb(x)\n        return x + positions\n\n\n   \nplt.style.use('ggplot')\n\ndef plot_history(history):\n    acc = history.history['accuracy']\n    val_acc = history.history['val_accuracy']\n    loss = history.history['loss']\n    val_loss = history.history['val_loss']\n    x = range(1, len(acc) + 1)\n\n    plt.figure(figsize=(12, 5))\n    plt.subplot(1, 2, 1)\n    plt.plot(x, acc, 'b', label='Training acc')\n    plt.plot(x, val_acc, 'r', label='Validation acc')\n    plt.title('Training and validation accuracy')\n    plt.legend()\n    plt.subplot(1, 2, 2)\n    plt.plot(x, loss, 'b', label='Training loss')\n    plt.plot(x, val_loss, 'r', label='Validation loss')\n    plt.title('Training and validation loss')\n    plt.legend()\n    \n\ndef _data_generator(x, y, num_features, batch_size):\n    \"\"\"Generates batches of vectorized texts for training/validation.\n\n    # Arguments\n        x: np.matrix, feature matrix.\n        y: np.ndarray, labels.\n        num_features: int, number of features.\n        batch_size: int, number of samples per batch.\n\n    # Returns\n        Yields feature and label data in batches.\n    \"\"\"\n    num_samples = x.shape[0]\n    num_batches = num_samples // batch_size\n    if num_samples % batch_size:\n        num_batches += 1\n\n    while 1:\n        for i in range(num_batches):\n            start_idx = i * batch_size\n            end_idx = (i + 1) * batch_size\n            if end_idx > num_samples:\n                end_idx = num_samples\n            x_batch = x[start_idx:end_idx]\n            y_batch = y[start_idx:end_idx]\n            yield x_batch, y_batch\n    ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jigsaw_train_df = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\njigsaw_unbias_df = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(jigsaw_train_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(jigsaw_unbias_df)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(jigsaw_train_df.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jigsaw_unbias_df = jigsaw_unbias_df.rename(columns={\"severe_toxicity\":\"severe_toxic\",\"identity_attack\":\"identity_hate\"})","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(jigsaw_unbias_df.columns)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jigsaw_df = pd.concat([jigsaw_train_df,jigsaw_unbias_df])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(len(jigsaw_df))\njigsaw_df = jigsaw_df.dropna(subset=[\"comment_text\"])\njigsaw_df = jigsaw_df.drop_duplicates(subset=[\"comment_text\"])\nprint(len(jigsaw_df))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jigsaw_df.insult = np.rint(jigsaw_df.insult)\njigsaw_df.toxic = np.rint(jigsaw_df.toxic)\njigsaw_df.severe_toxic = np.rint(jigsaw_df.severe_toxic)\njigsaw_df.obscene = np.rint(jigsaw_df.obscene)\njigsaw_df.identity_hate = np.rint(jigsaw_df.identity_hate)\njigsaw_df.threat = np.rint(jigsaw_df.threat)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"jigsaw_df.insult.unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Insults","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"text_train, text_val, y_train, y_val = model_selection.train_test_split(jigsaw_df[\"comment_text\"],jigsaw_df[\"insult\"],test_size=0.3)\nprint(\"type(y_train)\",type(y_train))\ny_train = y_train.to_numpy()\ny_val = y_val.to_numpy()\nprint(\"type(y_train)\",type(y_train))\nfrom keras.preprocessing.text import Tokenizer\n# Tokenize and transform to integer index\ntokenizer = Tokenizer(oov_token=True,num_words=10000)\ntokenizer.fit_on_texts(text_train)\n\nX_train = tokenizer.texts_to_sequences(text_train)\nX_val = tokenizer.texts_to_sequences(text_val)\n\nvocab_size = 10000 #len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index\nmaxlen = 128 #max(len(x) for x in X_train) # longest text in train set\nbatch_size = 32\n\nprint(\"vocab_size\",vocab_size)\nprint(\"maxlen\",maxlen)\n\n# Add pading to ensure all vectors have same dimensionality\n\nprint(len(X_train), \"Training sequences\")\nprint(len(X_val), \"Validation sequences\")\nx_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=maxlen)\nx_val = keras.preprocessing.sequence.pad_sequences(X_val, maxlen=maxlen)\nprint(\"type(x_train)\",type(x_train))\nx_train = np.asmatrix(x_train)\nprint(\"type(x_train)\",type(x_train))\nx_val = np.asmatrix(x_val)\n\ntraining_generator = _data_generator(\n    x_train, y_train, maxlen, batch_size)\nvalidation_generator = _data_generator(\n    x_val, y_val, maxlen, batch_size)\n\n\n# Get number of training steps. This indicated the number of steps it takes\n# to cover all samples in one epoch.\nsteps_per_epoch = x_train.shape[0] // batch_size\nif x_train.shape[0] % batch_size:\n    steps_per_epoch += 1\n\n\n# Get number of validation steps.\nvalidation_steps = x_val.shape[0] // batch_size\nif x_val.shape[0] % batch_size:\n    validation_steps += 1\n\n\nembed_dim = 128  # Embedding size for each token\nnum_heads = 4  # Number of attention heads\nff_dim = 128  # Hidden layer size in feed forward network inside transformer\n\n## Create classifier model using transformer layer\n\n#Transformer layer outputs one vector for each time step of our input sequence.\n#Here, we take the mean across all time steps and\n#use a feed forward network on top of it to classify text.\n\ninputs = layers.Input(shape=(maxlen,))\nembedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\nx = embedding_layer(inputs)\ntransformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = keras.Model(inputs=inputs, outputs=outputs)\n\nmodel.compile(\"adam\", \"binary_crossentropy\", metrics=[\"accuracy\"])\n# Train and validate model.\nhistory = model.fit_generator(\n    generator=training_generator,\n    steps_per_epoch=steps_per_epoch,\n    validation_data=validation_generator,\n    validation_steps=validation_steps,\n    epochs=2)  # Logs once per epoch.\n\n#model.save(\"/content/drive/My Drive/Colab Notebooks/CB_Data/transformer_block_Jigsaw_insults\")\n\n%matplotlib inline\nplot_history(history)\n\nfrom sklearn.metrics import roc_auc_score,f1_score\nimport numpy as np\npredicted_labels = model.predict(x_val)\nprint(\"AUC score\", roc_auc_score(np.rint(y_val),predicted_labels))\nprint(\"F1 score\", f1_score(np.rint(y_val),np.rint(predicted_labels)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"##Toxicity","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"text_train, text_val, y_train, y_val = model_selection.train_test_split(jigsaw_df[\"comment_text\"],jigsaw_df[\"toxic\"],test_size=0.3)\nprint(\"type(y_train)\",type(y_train))\ny_train = y_train.to_numpy()\ny_val = y_val.to_numpy()\nprint(\"type(y_train)\",type(y_train))\nfrom keras.preprocessing.text import Tokenizer\n# Tokenize and transform to integer index\ntokenizer = Tokenizer(oov_token=True,num_words=10000)\ntokenizer.fit_on_texts(text_train)\n\nX_train = tokenizer.texts_to_sequences(text_train)\nX_val = tokenizer.texts_to_sequences(text_val)\n\nvocab_size = 10000 #len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index\nmaxlen = 200 #max(len(x) for x in X_train) # longest text in train set\nbatch_size = 32\n\nprint(\"vocab_size\",vocab_size)\nprint(\"maxlen\",maxlen)\n\n# Add pading to ensure all vectors have same dimensionality\n\nprint(len(X_train), \"Training sequences\")\nprint(len(X_val), \"Validation sequences\")\nx_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=maxlen)\nx_val = keras.preprocessing.sequence.pad_sequences(X_val, maxlen=maxlen)\nprint(\"type(x_train)\",type(x_train))\nx_train = np.asmatrix(x_train)\nprint(\"type(x_train)\",type(x_train))\nx_val = np.asmatrix(x_val)\n\ntraining_generator = _data_generator(\n    x_train, y_train, maxlen, batch_size)\nvalidation_generator = _data_generator(\n    x_val, y_val, maxlen, batch_size)\n\n\n# Get number of training steps. This indicated the number of steps it takes\n# to cover all samples in one epoch.\nsteps_per_epoch = x_train.shape[0] // batch_size\nif x_train.shape[0] % batch_size:\n    steps_per_epoch += 1\n\n\n# Get number of validation steps.\nvalidation_steps = x_val.shape[0] // batch_size\nif x_val.shape[0] % batch_size:\n    validation_steps += 1\n\n\nembed_dim = 128  # Embedding size for each token\nnum_heads = 4  # Number of attention heads\nff_dim = 128  # Hidden layer size in feed forward network inside transformer\n\n## Create classifier model using transformer layer\n\n#Transformer layer outputs one vector for each time step of our input sequence.\n#Here, we take the mean across all time steps and\n#use a feed forward network on top of it to classify text.\n\ninputs = layers.Input(shape=(maxlen,))\nembedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\nx = embedding_layer(inputs)\ntransformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = keras.Model(inputs=inputs, outputs=outputs)\n\nmodel.compile(\"adam\", \"binary_crossentropy\", metrics=[\"accuracy\"])\n# Train and validate model.\nhistory = model.fit_generator(\n    generator=training_generator,\n    steps_per_epoch=steps_per_epoch,\n    validation_data=validation_generator,\n    validation_steps=validation_steps,\n    epochs=2)  # Logs once per epoch.\n\n#model.save(\"/content/drive/My Drive/Colab Notebooks/CB_Data/transformer_block_Jigsaw_insults\")\n\n%matplotlib inline\nplot_history(history)\n\nfrom sklearn.metrics import roc_auc_score,f1_score\nimport numpy as np\npredicted_labels = model.predict(x_val)\nprint(\"AUC score\", roc_auc_score(np.rint(y_val),predicted_labels))\nprint(\"F1 score\", f1_score(np.rint(y_val),np.rint(predicted_labels)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## obscene","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"text_train, text_val, y_train, y_val = model_selection.train_test_split(jigsaw_df[\"comment_text\"],jigsaw_df[\"obscene\"],test_size=0.3)\nprint(\"type(y_train)\",type(y_train))\ny_train = y_train.to_numpy()\ny_val = y_val.to_numpy()\nprint(\"type(y_train)\",type(y_train))\nfrom keras.preprocessing.text import Tokenizer\n# Tokenize and transform to integer index\ntokenizer = Tokenizer(oov_token=True,num_words=10000)\ntokenizer.fit_on_texts(text_train)\n\nX_train = tokenizer.texts_to_sequences(text_train)\nX_val = tokenizer.texts_to_sequences(text_val)\n\nvocab_size = 10000 #len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index\nmaxlen = 200 #max(len(x) for x in X_train) # longest text in train set\nbatch_size = 32\n\nprint(\"vocab_size\",vocab_size)\nprint(\"maxlen\",maxlen)\n\n# Add pading to ensure all vectors have same dimensionality\n\nprint(len(X_train), \"Training sequences\")\nprint(len(X_val), \"Validation sequences\")\nx_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=maxlen)\nx_val = keras.preprocessing.sequence.pad_sequences(X_val, maxlen=maxlen)\nprint(\"type(x_train)\",type(x_train))\nx_train = np.asmatrix(x_train)\nprint(\"type(x_train)\",type(x_train))\nx_val = np.asmatrix(x_val)\n\ntraining_generator = _data_generator(\n    x_train, y_train, maxlen, batch_size)\nvalidation_generator = _data_generator(\n    x_val, y_val, maxlen, batch_size)\n\n\n# Get number of training steps. This indicated the number of steps it takes\n# to cover all samples in one epoch.\nsteps_per_epoch = x_train.shape[0] // batch_size\nif x_train.shape[0] % batch_size:\n    steps_per_epoch += 1\n\n\n# Get number of validation steps.\nvalidation_steps = x_val.shape[0] // batch_size\nif x_val.shape[0] % batch_size:\n    validation_steps += 1\n\n\nembed_dim = 128  # Embedding size for each token\nnum_heads = 4  # Number of attention heads\nff_dim = 128  # Hidden layer size in feed forward network inside transformer\n\n## Create classifier model using transformer layer\n\n#Transformer layer outputs one vector for each time step of our input sequence.\n#Here, we take the mean across all time steps and\n#use a feed forward network on top of it to classify text.\n\ninputs = layers.Input(shape=(maxlen,))\nembedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\nx = embedding_layer(inputs)\ntransformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = keras.Model(inputs=inputs, outputs=outputs)\n\nmodel.compile(\"adam\", \"binary_crossentropy\", metrics=[\"accuracy\"])\n# Train and validate model.\nhistory = model.fit_generator(\n    generator=training_generator,\n    steps_per_epoch=steps_per_epoch,\n    validation_data=validation_generator,\n    validation_steps=validation_steps,\n    epochs=2)  # Logs once per epoch.\n\n#model.save(\"/content/drive/My Drive/Colab Notebooks/CB_Data/transformer_block_Jigsaw_insults\")\n\n%matplotlib inline\nplot_history(history)\n\nfrom sklearn.metrics import roc_auc_score,f1_score\nimport numpy as np\npredicted_labels = model.predict(x_val)\nprint(\"AUC score\", roc_auc_score(np.rint(y_val),predicted_labels))\nprint(\"F1 score\", f1_score(np.rint(y_val),np.rint(predicted_labels)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## threat","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"text_train, text_val, y_train, y_val = model_selection.train_test_split(jigsaw_df[\"comment_text\"],jigsaw_df[\"threat\"],test_size=0.3)\nprint(\"type(y_train)\",type(y_train))\ny_train = y_train.to_numpy()\ny_val = y_val.to_numpy()\nprint(\"type(y_train)\",type(y_train))\nfrom keras.preprocessing.text import Tokenizer\n# Tokenize and transform to integer index\ntokenizer = Tokenizer(oov_token=True,num_words=10000)\ntokenizer.fit_on_texts(text_train)\n\nX_train = tokenizer.texts_to_sequences(text_train)\nX_val = tokenizer.texts_to_sequences(text_val)\n\nvocab_size = 10000 #len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index\nmaxlen = 200 #max(len(x) for x in X_train) # longest text in train set\nbatch_size = 32\n\nprint(\"vocab_size\",vocab_size)\nprint(\"maxlen\",maxlen)\n\n# Add pading to ensure all vectors have same dimensionality\n\nprint(len(X_train), \"Training sequences\")\nprint(len(X_val), \"Validation sequences\")\nx_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=maxlen)\nx_val = keras.preprocessing.sequence.pad_sequences(X_val, maxlen=maxlen)\nprint(\"type(x_train)\",type(x_train))\nx_train = np.asmatrix(x_train)\nprint(\"type(x_train)\",type(x_train))\nx_val = np.asmatrix(x_val)\n\ntraining_generator = _data_generator(\n    x_train, y_train, maxlen, batch_size)\nvalidation_generator = _data_generator(\n    x_val, y_val, maxlen, batch_size)\n\n\n# Get number of training steps. This indicated the number of steps it takes\n# to cover all samples in one epoch.\nsteps_per_epoch = x_train.shape[0] // batch_size\nif x_train.shape[0] % batch_size:\n    steps_per_epoch += 1\n\n\n# Get number of validation steps.\nvalidation_steps = x_val.shape[0] // batch_size\nif x_val.shape[0] % batch_size:\n    validation_steps += 1\n\n\nembed_dim = 128  # Embedding size for each token\nnum_heads = 4  # Number of attention heads\nff_dim = 128  # Hidden layer size in feed forward network inside transformer\n\n## Create classifier model using transformer layer\n\n#Transformer layer outputs one vector for each time step of our input sequence.\n#Here, we take the mean across all time steps and\n#use a feed forward network on top of it to classify text.\n\ninputs = layers.Input(shape=(maxlen,))\nembedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\nx = embedding_layer(inputs)\ntransformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = keras.Model(inputs=inputs, outputs=outputs)\n\nmodel.compile(\"adam\", \"binary_crossentropy\", metrics=[\"accuracy\"])\n# Train and validate model.\nhistory = model.fit_generator(\n    generator=training_generator,\n    steps_per_epoch=steps_per_epoch,\n    validation_data=validation_generator,\n    validation_steps=validation_steps,\n    epochs=2)  # Logs once per epoch.\n\n#model.save(\"/content/drive/My Drive/Colab Notebooks/CB_Data/transformer_block_Jigsaw_insults\")\n\n%matplotlib inline\nplot_history(history)\n\nfrom sklearn.metrics import roc_auc_score,f1_score\nimport numpy as np\npredicted_labels = model.predict(x_val)\nprint(\"AUC score\", roc_auc_score(np.rint(y_val),predicted_labels))\nprint(\"F1 score\", f1_score(np.rint(y_val),np.rint(predicted_labels)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##identity hate","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"text_train, text_val, y_train, y_val = model_selection.train_test_split(jigsaw_df[\"comment_text\"],jigsaw_df[\"identity_hate\"],test_size=0.3)\nprint(\"type(y_train)\",type(y_train))\ny_train = y_train.to_numpy()\ny_val = y_val.to_numpy()\nprint(\"type(y_train)\",type(y_train))\nfrom keras.preprocessing.text import Tokenizer\n# Tokenize and transform to integer index\ntokenizer = Tokenizer(oov_token=True,num_words=10000)\ntokenizer.fit_on_texts(text_train)\n\nX_train = tokenizer.texts_to_sequences(text_train)\nX_val = tokenizer.texts_to_sequences(text_val)\n\nvocab_size = 10000 #len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index\nmaxlen = 200 #max(len(x) for x in X_train) # longest text in train set\nbatch_size = 32\n\nprint(\"vocab_size\",vocab_size)\nprint(\"maxlen\",maxlen)\n\n# Add pading to ensure all vectors have same dimensionality\n\nprint(len(X_train), \"Training sequences\")\nprint(len(X_val), \"Validation sequences\")\nx_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=maxlen)\nx_val = keras.preprocessing.sequence.pad_sequences(X_val, maxlen=maxlen)\nprint(\"type(x_train)\",type(x_train))\nx_train = np.asmatrix(x_train)\nprint(\"type(x_train)\",type(x_train))\nx_val = np.asmatrix(x_val)\n\ntraining_generator = _data_generator(\n    x_train, y_train, maxlen, batch_size)\nvalidation_generator = _data_generator(\n    x_val, y_val, maxlen, batch_size)\n\n\n# Get number of training steps. This indicated the number of steps it takes\n# to cover all samples in one epoch.\nsteps_per_epoch = x_train.shape[0] // batch_size\nif x_train.shape[0] % batch_size:\n    steps_per_epoch += 1\n\n\n# Get number of validation steps.\nvalidation_steps = x_val.shape[0] // batch_size\nif x_val.shape[0] % batch_size:\n    validation_steps += 1\n\n\nembed_dim = 128  # Embedding size for each token\nnum_heads = 4  # Number of attention heads\nff_dim = 128  # Hidden layer size in feed forward network inside transformer\n\n## Create classifier model using transformer layer\n\n#Transformer layer outputs one vector for each time step of our input sequence.\n#Here, we take the mean across all time steps and\n#use a feed forward network on top of it to classify text.\n\ninputs = layers.Input(shape=(maxlen,))\nembedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\nx = embedding_layer(inputs)\ntransformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = keras.Model(inputs=inputs, outputs=outputs)\n\nmodel.compile(\"adam\", \"binary_crossentropy\", metrics=[\"accuracy\"])\n# Train and validate model.\nhistory = model.fit_generator(\n    generator=training_generator,\n    steps_per_epoch=steps_per_epoch,\n    validation_data=validation_generator,\n    validation_steps=validation_steps,\n    epochs=2)  # Logs once per epoch.\n\n#model.save(\"/content/drive/My Drive/Colab Notebooks/CB_Data/transformer_block_Jigsaw_insults\")\n\n%matplotlib inline\nplot_history(history)\n\nfrom sklearn.metrics import roc_auc_score,f1_score\nimport numpy as np\npredicted_labels = model.predict(x_val)\nprint(\"AUC score\", roc_auc_score(np.rint(y_val),predicted_labels))\nprint(\"F1 score\", f1_score(np.rint(y_val),np.rint(predicted_labels)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"##severe toxicity","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"text_train, text_val, y_train, y_val = model_selection.train_test_split(jigsaw_df[\"comment_text\"],jigsaw_df[\"severe_toxic\"],test_size=0.3)\nprint(\"type(y_train)\",type(y_train))\ny_train = y_train.to_numpy()\ny_val = y_val.to_numpy()\nprint(\"type(y_train)\",type(y_train))\nfrom keras.preprocessing.text import Tokenizer\n# Tokenize and transform to integer index\ntokenizer = Tokenizer(oov_token=True,num_words=10000)\ntokenizer.fit_on_texts(text_train)\n\nX_train = tokenizer.texts_to_sequences(text_train)\nX_val = tokenizer.texts_to_sequences(text_val)\n\nvocab_size = 10000 #len(tokenizer.word_index) + 1  # Adding 1 because of reserved 0 index\nmaxlen = 200 #max(len(x) for x in X_train) # longest text in train set\nbatch_size = 32\n\nprint(\"vocab_size\",vocab_size)\nprint(\"maxlen\",maxlen)\n\n# Add pading to ensure all vectors have same dimensionality\n\nprint(len(X_train), \"Training sequences\")\nprint(len(X_val), \"Validation sequences\")\nx_train = keras.preprocessing.sequence.pad_sequences(X_train, maxlen=maxlen)\nx_val = keras.preprocessing.sequence.pad_sequences(X_val, maxlen=maxlen)\nprint(\"type(x_train)\",type(x_train))\nx_train = np.asmatrix(x_train)\nprint(\"type(x_train)\",type(x_train))\nx_val = np.asmatrix(x_val)\n\ntraining_generator = _data_generator(\n    x_train, y_train, maxlen, batch_size)\nvalidation_generator = _data_generator(\n    x_val, y_val, maxlen, batch_size)\n\n\n# Get number of training steps. This indicated the number of steps it takes\n# to cover all samples in one epoch.\nsteps_per_epoch = x_train.shape[0] // batch_size\nif x_train.shape[0] % batch_size:\n    steps_per_epoch += 1\n\n\n# Get number of validation steps.\nvalidation_steps = x_val.shape[0] // batch_size\nif x_val.shape[0] % batch_size:\n    validation_steps += 1\n\n\nembed_dim = 128  # Embedding size for each token\nnum_heads = 4  # Number of attention heads\nff_dim = 128  # Hidden layer size in feed forward network inside transformer\n\n## Create classifier model using transformer layer\n\n#Transformer layer outputs one vector for each time step of our input sequence.\n#Here, we take the mean across all time steps and\n#use a feed forward network on top of it to classify text.\n\ninputs = layers.Input(shape=(maxlen,))\nembedding_layer = TokenAndPositionEmbedding(maxlen, vocab_size, embed_dim)\nx = embedding_layer(inputs)\ntransformer_block = TransformerBlock(embed_dim, num_heads, ff_dim)\nx = transformer_block(x)\nx = layers.GlobalAveragePooling1D()(x)\nx = layers.Dropout(0.1)(x)\nx = layers.Dense(20, activation=\"relu\")(x)\nx = layers.Dropout(0.1)(x)\noutputs = layers.Dense(1, activation=\"sigmoid\")(x)\n\nmodel = keras.Model(inputs=inputs, outputs=outputs)\n\nmodel.compile(\"adam\", \"binary_crossentropy\", metrics=[\"accuracy\"])\n# Train and validate model.\nhistory = model.fit_generator(\n    generator=training_generator,\n    steps_per_epoch=steps_per_epoch,\n    validation_data=validation_generator,\n    validation_steps=validation_steps,\n    epochs=2)  # Logs once per epoch.\n\n#model.save(\"/content/drive/My Drive/Colab Notebooks/CB_Data/transformer_block_Jigsaw_insults\")\n\n%matplotlib inline\nplot_history(history)\n\nfrom sklearn.metrics import roc_auc_score,f1_score\nimport numpy as np\npredicted_labels = model.predict(x_val)\nprint(\"AUC score\", roc_auc_score(np.rint(y_val),predicted_labels))\nprint(\"F1 score\", f1_score(np.rint(y_val),np.rint(predicted_labels)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}