{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"#import necessary libraries\n\nimport numpy as np\nimport pandas as pd \nimport tensorflow as tf\nimport re\nfrom nltk.corpus import stopwords\n\nfrom tensorflow.keras import regularizers, initializers, optimizers, callbacks\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom keras.utils.np_utils import to_categorical\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.models import Model\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"# read datasets\n\ntrn1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\ntrn2 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv')\n\n\nval = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"project_test = pd.read_csv('/kaggle/input/test-sentences-jigsaw/testtest.csv',header=None,names=['sentence', 'label'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# data cleaning step: labels in trn2 not represented as binary integers\n\ntrn2['toxic'] = trn2.toxic.round().astype(int)\ntrn2_tox = trn2[trn2['toxic'] == 1]\ntrn2_ok = trn2[trn2['toxic'] == 0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# combine into one training set\n\ntrn =  pd.concat([trn1[['comment_text', 'toxic']],trn2_tox[['comment_text', 'toxic']],trn2_ok[['comment_text', 'toxic']]])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# condense the training set since there are over a million examples\n\ntrn = trn.sample(n=300000)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Data Cleaning:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# to be used for exploratory visualizations\n\ntrn_toxic = trn[trn['toxic'] == 1]\ntrn_nt = trn[trn['toxic'] ==0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(trn_toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# remove empty comments\n\ntrn = trn.where(trn['comment_text'] != \"\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# more standardizing\n\nlabels = ['toxic', 'not_toxic']\nclasses = trn['toxic'].values\ntrn_comments = trn['comment_text']\ntrn_comments = list(trn_comments)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# strip invalid characters and remove stopwords\n\ndef process_text(text, remove_stopwords = True):\n    output = \"\"\n    text = str(text).replace(\"\\n\", \"\")\n    text = re.sub(r'[^\\w\\s]','',text).lower()\n    if remove_stopwords:\n        text = text.split(\" \")\n        for word in text:\n            if word not in stopwords.words(\"english\"):\n                output = output + \" \" + word\n    else:\n        output = text\n    return str(output.strip())[1:-3].replace(\"  \", \" \")\n    \ntexts = [] \n\nfor line in trn_comments: \n    texts.append(process_text(line))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"EDA & Summary Statistics"},{"metadata":{"trusted":true},"cell_type":"code","source":"# make word cloud for toxic comments\n\nfrom wordcloud import WordCloud\nimport plotly.express as px\n\ndef nonan(x):\n    if type(x) == str:\n        return x.replace(\"\\n\", \"\")\n    else:\n        return \"\"\n\ntext = ' '.join([nonan(abstract) for abstract in trn_toxic[\"comment_text\"]])\n\nwordcloud1 = WordCloud(max_font_size=None, background_color='white', collocations=False,width=1200, height=1000).generate(text)\nfig1 = px.imshow(wordcloud1)\nfig1.update_layout(title_text='Common words in toxic comments')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make word cloud for non-toxic comments\n\ntext = ' '.join([nonan(abstract) for abstract in trn_nt[\"comment_text\"]])\n\nwordcloud2 = WordCloud(max_font_size=None, background_color='white', collocations=False,\n                      width=1200, height=1000).generate(text)\nfig2 = px.imshow(wordcloud2)\nfig2.update_layout(title_text='Common words in non-toxic comments')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# percentage of toxic comments in training set\n\nimport seaborn as sns\nsns.barplot(x=['Not Toxic', 'Toxic'], y=trn['toxic'].value_counts())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# hyperparams\n\nMAX_NB_WORDS = 100000    \nMAX_SEQUENCE_LENGTH = 200 \nVALIDATION_SPLIT = 0.2  \nEMBEDDING_DIM = 100      \nGLOVE_DIR = '/kaggle/input/glove6b100dtxt/glove.6B.100d.txt'","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# tokenize text and create dictionary\n\ntokenizer = Tokenizer(num_words=MAX_NB_WORDS)\ntokenizer.fit_on_texts(texts)\nsequences = tokenizer.texts_to_sequences(texts)\nword_index = tokenizer.word_index\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# pad sequences to standardize length\n\ndata = pad_sequences(sequences, padding = 'post', maxlen = MAX_SEQUENCE_LENGTH)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split into train-validation\n\nindices = np.arange(data.shape[0])\nnp.random.shuffle(indices)\ndata = data[indices]\nlabels = classes[indices]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# split into train-validation\n\nnum_validation_samples = int(VALIDATION_SPLIT*data.shape[0])\nx_train = data[: -num_validation_samples]\ny_train = labels[: -num_validation_samples]\nx_val = data[-num_validation_samples: ]\ny_val = labels[-num_validation_samples: ]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# upload pre-trained word embedding and set up embedding matrix for embedding layer\n\nembeddings_index = {}\nf = open(GLOVE_DIR)\nfor line in f:\n    values = line.split()\n    word = values[0]\n    embeddings_index[word] = np.asarray(values[1:], dtype='float32')\nf.close()\n\nembedding_matrix = np.random.random((len(word_index) + 1, EMBEDDING_DIM))\nfor word, i in word_index.items():\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"LSTM Implementation:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def lstm():\n    sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n    embedding_layer = Embedding(len(word_index) + 1,\n                               EMBEDDING_DIM,\n                               weights = [embedding_matrix],\n                               input_length = MAX_SEQUENCE_LENGTH,\n                               trainable=False,\n                               name = 'embeddings')\n    embedded_sequences = embedding_layer(sequence_input)\n    x = LSTM(25, return_sequences=True,name='lstm_layer')(embedded_sequences)\n    x = GlobalMaxPool1D()(x)\n    x = Dropout(0.5)(x)\n    x = Dense(20, activation=\"relu\")(x)\n    x = Dropout(0.5)(x)\n    preds = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(sequence_input, preds)\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lstm = lstm()\nlstm.compile(loss = 'binary_crossentropy',\n             optimizer='adam',\n             metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.utils.plot_model(lstm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print('Training progress:')\nlstm_history = lstm.fit(x_train, y_train, epochs = 15, batch_size=32, validation_data=(x_val, y_val))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nloss_lstm = lstm_history.history['loss']\nval_loss_lstm = lstm_history.history['val_loss']\nepochs = range(1, len(loss_lstm)+1)\nplt.plot(epochs, loss_lstm, label='Training loss')\nplt.plot(epochs, val_loss_lstm, label='Validation loss')\nplt.title('Training and validation loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_curve\n\ny_hat_lstm = lstm.predict(x_val).ravel()\nfpr_lstm, tpr_lstm, thresholds_lstm = roc_curve(y_val, y_hat_lstm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import auc\nauc_lstm = auc(fpr_lstm, tpr_lstm)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"auc_lstm","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.figure(1)\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(fpr_lstm, tpr_lstm, label='LSTM'.format(auc_lstm))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.show()\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_hat = np.around(y_hat_lstm, decimals=0, out=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import classification_report","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(y_val, y_hat, digits=2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Make Predictions on LSTM:"},{"metadata":{"trusted":true},"cell_type":"code","source":"# define test sentences for project\nX = project_test['sentence']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lstm_predictions_project = lstm.predict(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(lstm_predictions_project)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"CNN Implementation:"},{"metadata":{"trusted":true},"cell_type":"code","source":"def cnn():\n    \n    sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32')\n    embedding_layer = Embedding(len(word_index) + 1,\n                               EMBEDDING_DIM,\n                               weights = [embedding_matrix],\n                               input_length = MAX_SEQUENCE_LENGTH,\n                               trainable=False,\n                               name = 'embeddings')\n    \n    embedded_sequences = embedding_layer(sequence_input)\n    convs = []\n    filter_sizes = [2,3,4]\n    for filter_size in filter_sizes:\n        l_conv = Conv1D(filters=30, \n                        kernel_size=filter_size, \n                        activation='relu')(embedded_sequences)\n        l_pool = GlobalMaxPooling1D()(l_conv)\n        convs.append(l_pool)\n    l_merge = concatenate(convs, axis=1)\n    x = Dropout(0.5)(l_merge)  \n    x = Dense(20, activation='relu')(x)\n    x = Dropout(0.5)(x)\n    preds = Dense(1, activation='sigmoid')(x)\n    model = Model(sequence_input, preds)\n    model.compile(loss='binary_crossentropy',\n                  optimizer='adam',\n                  metrics=['acc'])\n    model.summary()\n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn = cnn()\ntf.keras.utils.plot_model(cnn)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn = cnn()\ncnn.compile(loss = 'binary_crossentropy',\n             optimizer='adam',\n             metrics = ['accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn_history = cnn.fit(x_train, y_train, epochs = 15, batch_size=32, validation_data=(x_val, y_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\n\nloss_cnn = cnn_history.history['loss']\nval_loss_cnn = cnn_history.history['val_loss']\nepochs = range(1, len(loss_cnn)+1)\nplt.plot(epochs, loss_cnn, label='Training loss')\nplt.plot(epochs, val_loss_cnn, label='Validation loss')\nplt.title('Training and validation loss')\nplt.xlabel('Epochs')\nplt.ylabel('Loss')\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_curve\n\ny_hat_cnn = cnn.predict(x_val).ravel()\nfpr_cnn, tpr_cnn, thresholds_cnn = roc_curve(y_val, y_hat_cnn)\n\nfrom sklearn.metrics import auc\nauc_cnn = auc(fpr_cnn, tpr_cnn)\n\nplt.figure(1)\nplt.plot([0, 1], [0, 1], 'k--')\nplt.plot(fpr_cnn, tpr_cnn, label='CNN'.format(auc_cnn))\nplt.xlabel('False positive rate')\nplt.ylabel('True positive rate')\nplt.title('ROC curve')\nplt.legend(loc='best')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"auc_cnn","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_hat_cnn = np.around(y_hat_cnn, decimals=0, out=None)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(classification_report(y_val, y_hat_cnn, digits=2))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Make predictions on CNN:"},{"metadata":{"trusted":true},"cell_type":"code","source":"cnn_predictions_project = cnn.predict(X)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(cnn_predictions_project)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}