{"cells":[{"metadata":{"_uuid":"5a6af4c3-f378-4687-ac26-b18828ad56d5","_cell_guid":"1ffaa7fa-e615-468c-94fa-ff14ef4d31d3","trusted":true},"cell_type":"markdown","source":"# LSTM with pre-trained GloVe embeddings "},{"metadata":{"_uuid":"8eb30a08-f619-43a1-a44a-54889883ff46","_cell_guid":"f38b3c8a-37cb-4257-b6f6-2d17356ec7b6","trusted":true},"cell_type":"code","source":"# # Load packages\n\n# Ignore warnings\nimport warnings\n\ndef warn(*args, **kwargs):\n    pass\n\nwarnings.warn = warn\n\nimport os\nimport numpy as np\nimport pandas as pd\nimport time\n\nimport keras\nfrom keras import *\nfrom keras import layers\nfrom keras.layers import Embedding, LSTM, Dense, Dropout, Bidirectional\nfrom keras.models import Model\nfrom keras.preprocessing import *\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score, confusion_matrix\nfrom sklearn.model_selection import train_test_split\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nimport re\nfrom io import StringIO","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2eac7e1e-7127-43db-8634-101a576eb022","_cell_guid":"8f68c306-9279-4e12-9671-8a2f9c3e9d91","trusted":true},"cell_type":"markdown","source":"# Helper functions"},{"metadata":{"_uuid":"77be22b7-4152-4fd0-b12a-bd93ef40b3f9","_cell_guid":"8a1ccf40-7cb3-4595-8320-649b13fdc047","trusted":true},"cell_type":"code","source":"class RocCallback(Callback):\n    def __init__(self,training_data,validation_data):\n        self.x = training_data[0]\n        self.y = training_data[1]\n        self.x_val = validation_data[0]\n        self.y_val = validation_data[1]\n    \n    def on_train_begin(self, logs={}):\n        return\n\n    def on_train_end(self, logs={}):\n        return\n\n    def on_epoch_begin(self, epoch, logs={}):\n        return\n\n    def on_epoch_end(self, epoch, logs={}):\n        y_pred_train = self.model.predict_proba(self.x)\n        roc_train = roc_auc_score(self.y, y_pred_train)\n        y_pred_val = self.model.predict_proba(self.x_val)\n        roc_val = roc_auc_score(self.y_val, y_pred_val)\n        print('\\rroc-auc_train: %s - roc-auc_val: %s' % (str(round(roc_train,4)),str(round(roc_val,4))),end=100*' '+'\\n')\n        return\n\n    def on_batch_begin(self, batch, logs={}):\n        return\n\n    def on_batch_end(self, batch, logs={}):\n        return\n\n\n    \nPLOT_FONT_SIZE = 10    #font size for axis of plots\n\n#define helper function for confusion matrix\n\ndef displayConfusionMatrix(confusionMatrix):\n    \"\"\"Confusion matrix plot\"\"\"\n    \n    confusionMatrix = np.transpose(confusionMatrix)\n    \n    ## calculate class level precision and recall from confusion matrix\n    precisionLow = round((confusionMatrix[0][0] / (confusionMatrix[0][0] + confusionMatrix[0][1]))*100, 1)\n    precisionHigh = round((confusionMatrix[1][1] / (confusionMatrix[1][0] + confusionMatrix[1][1]))*100, 1)\n    recallLow = round((confusionMatrix[0][0] / (confusionMatrix[0][0] + confusionMatrix[1][0]))*100, 1)\n    recallHigh = round((confusionMatrix[1][1] / (confusionMatrix[0][1] + confusionMatrix[1][1]))*100, 1)\n\n    ## show heatmap\n    plt.imshow(confusionMatrix, interpolation='nearest',cmap=plt.cm.Blues,vmin=0, vmax=100)\n    \n    ## axis labeling\n    xticks = np.array([0,1])\n    plt.gca().set_xticks(xticks)\n    plt.gca().set_yticks(xticks)\n    plt.gca().set_xticklabels([\"Not Toxic \\n Recall=\" + str(recallLow), \"Toxic \\n Recall=\" + str(recallHigh)], fontsize=PLOT_FONT_SIZE)\n    plt.gca().set_yticklabels([\"Not Toxic \\n Precision=\" + str(precisionLow), \"Toxic \\n Precision=\" + str(precisionHigh)], fontsize=PLOT_FONT_SIZE)\n    plt.ylabel(\"Predicted Class\", fontsize=PLOT_FONT_SIZE)\n    plt.xlabel(\"Actual Class\", fontsize=PLOT_FONT_SIZE)\n        \n    ## add text in heatmap boxes\n    addText(xticks, xticks, confusionMatrix)\n    \ndef addText(xticks, yticks, results):\n    \"\"\"Add text in the plot\"\"\"\n    for i in range(len(yticks)):\n        for j in range(len(xticks)):\n            text = plt.text(j, i, results[i][j], ha=\"center\", va=\"center\", color=\"white\", size=PLOT_FONT_SIZE) ### size here is the size of text inside a single box in the heatmap","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"13a6d0e5-a307-49e6-8218-eab05e1bdba9","_cell_guid":"589e122e-c513-49cf-a76d-000d78cbf478","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"738370a4-5479-4028-b108-d3a4d461d26c","_cell_guid":"474e7c56-2047-4dc6-9523-c148bf9a45fc","trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"019aa8b5-0389-447c-b406-d1ecbcc9a2f4","_cell_guid":"b419d588-0ca3-4cd7-896b-fc0dd6f5e068","trusted":true},"cell_type":"code","source":"def lemmetize_data(data,field):\n    cleaned_texts = []\n    for text in data[field]: # Loop through the tokens (the words or symbols) \n        cleaned_text = text.lower()  # Convert the text to lower case\n        cleaned_text = ' '.join([word for word in cleaned_text.split() if word not in stopset])  # Keep only words that are not stopwords.\n        cleaned_text = ' '.join([wordnet_lemmatizer.lemmatize(word, pos='n') for word in cleaned_text.split()])  # Keep each noun's lemma.\n        cleaned_text = ' '.join([wordnet_lemmatizer.lemmatize(word, pos='v') for word in cleaned_text.split()])  # Keep each verb's lemma.\n        cleaned_text = re.sub(\"[^a-zA-Z]\",\" \", cleaned_text)  # Remove numbers and punctuation.\n        cleaned_text = ' '.join(cleaned_text.split())  # Remove white space.\n        cleaned_texts.append(cleaned_text) \n    data['cleanText'] = cleaned_texts","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1d531c95-8f14-4867-88ba-523d89dd43f8","_cell_guid":"84869af6-c3c1-4640-ab10-2cb483e54dcd","trusted":true},"cell_type":"code","source":"nltk.download('stopwords')\nnltk.download('wordnet')\n\nwordnet_lemmatizer = WordNetLemmatizer()\nstopset = list(set(stopwords.words('english')))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"aefd616d-dce8-4c39-b784-66783940aaf8","_cell_guid":"ec120490-5fef-4f4f-aabb-d172aba147c2","trusted":true},"cell_type":"markdown","source":"# Load data"},{"metadata":{"_uuid":"c452eb60-1946-40e9-afef-81085afeda81","_cell_guid":"bcb965fc-0eb5-4159-b392-9b9857dd0dbd","trusted":true},"cell_type":"code","source":"# load training data 1\ntrain_comment=pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\nlemmetize_data(train_comment,'comment_text')\ntrain_comment","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"0d341fac-d363-47b6-9aef-db6bec82e400","_cell_guid":"e67b42db-7775-4cb8-ad57-1f7a4ff9963e","trusted":true},"cell_type":"code","source":"# load validation data 1\nvalidGoogle=pd.read_csv('../input/val-en-df/validation_en.csv')\nlemmetize_data(validGoogle,'comment_text_en')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"97232533-caec-4364-a280-d0c2a7baa803","_cell_guid":"cad49668-3b24-4e90-897e-c2a2bded6808","trusted":true},"cell_type":"code","source":"# load validation data 2\nvalidYandex=pd.read_csv('../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv')\nlemmetize_data(validYandex,'translated')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7f0e894e-98bb-44de-8bfd-4e7cae84a46b","_cell_guid":"1e4360a9-1bee-449e-84e4-49e38bb8fb86","trusted":true},"cell_type":"code","source":"# append, when we calculate AUC it will reflect the average\nvalid = validGoogle.append(validYandex)\nvalid","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"20e8e2e7-7d45-43b6-8ee5-c96877b29960","_cell_guid":"8827890b-c053-4232-bb39-3ad5fda5ad4f","trusted":true},"cell_type":"code","source":"# load testing data (Translated via Google)\ntest_google=pd.read_csv('../input/test-en-df/test_en.csv')\nlemmetize_data(test_google,'content_en')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f4fd4320-1866-4b71-9f21-488ef6c9ccd3","_cell_guid":"799fe749-6772-4950-998a-afdfaf091b61","trusted":true},"cell_type":"code","source":"# load testing data (Translated via Yandex)\ntest_yandex=pd.read_csv('../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')\nlemmetize_data(test_yandex,'translated')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"68bf5c50-f354-4e28-af20-52f166a4697f","_cell_guid":"0b6bc136-383e-4d69-9a98-8dc1136196d7","trusted":true},"cell_type":"code","source":"def plotCases(data):\n    cases_count = data.value_counts(dropna=False)\n\n    # Plot  results \n    plt.figure(figsize=(6,6))\n    sns.barplot(x=cases_count.index, y=cases_count.values)\n    plt.ylabel('Texts', fontsize=12)\n    plt.xticks(range(len(cases_count.index)), ['Not', 'Toxic'])","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"08aa5cfe-4c87-413a-adac-55a59c8bfebd","_cell_guid":"6a286fbb-7680-4416-8e9e-4f9a832e85c3","trusted":true},"cell_type":"code","source":"train_labels = train_comment['toxic']\nvalid_labels = valid['toxic']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9c5a4c31-7adf-4295-a30d-33ddea33c52d","_cell_guid":"0ee8e73a-5912-4094-98e2-4b54632b9e7e","trusted":true},"cell_type":"code","source":"plotCases(train_labels)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"67b1b0e2-50fc-4bca-a6c0-177e95ff63bd","_cell_guid":"e864fed0-c626-4e92-85ec-d21b1b283e17","trusted":true},"cell_type":"code","source":"# the numbers here are doubled from the append above\nplotCases(valid_labels)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d277e9f3-5619-4c7f-b64c-d5033f6d7d06","_cell_guid":"27ea6894-f273-40b6-a8ad-09f2fb03ab01","trusted":true},"cell_type":"markdown","source":"# LSTM with pretrained"},{"metadata":{"_uuid":"ba8cffb3-c009-40b8-b8b8-f580cb0a246e","_cell_guid":"3653667a-7946-4d86-ba48-a33a86716a24","trusted":true},"cell_type":"markdown","source":"Prepare data:"},{"metadata":{"_uuid":"478fc91f-3de1-46ec-bba8-6ef10bb6c02f","_cell_guid":"aad3bb48-16bc-4555-8370-6cb0979d5216","trusted":true},"cell_type":"code","source":"# lemmetize\ntrain_texts = train_comment['cleanText']\n\nvalid_texts = valid['cleanText']\n\ntest_textsGoogle = test_google['cleanText']\ntest_textsYandex = test_yandex['cleanText']","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"04ecf58d-a478-410f-86ef-0e10f573c5b7","_cell_guid":"d5d7ad83-2978-46f5-bfff-1dcdcac0e612","trusted":true},"cell_type":"code","source":"# Define vocabulary size (you can tune this parameter and evaluate model performance)\nVOCABULARY_SIZE = 15000","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4eeede78-d308-41a0-8cdc-b56b54e041b9","_cell_guid":"8b6b9c38-8713-45ec-9393-17b32dceae67","trusted":true},"cell_type":"code","source":"# Create input feature arrays\ntokenizer = Tokenizer(num_words=VOCABULARY_SIZE)\ntokenizer.fit_on_texts(train_texts)\ntrain_texts","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"41f5d2af-87a1-402f-a3f9-4c5a7edee3a0","_cell_guid":"dcaae09b-ba76-4aaa-97ed-06862a30731d","trusted":true},"cell_type":"code","source":"# Convert words into word ids\nmeanLengthTrain = np.mean([len(item.split(\" \")) for item in train_texts])\nmeanLengthValid = np.mean([len(item.split(\" \")) for item in valid_texts])\nmeanLengthTestGoogle = np.mean([len(item.split(\" \")) for item in test_textsGoogle])\nmeanLengthTestYandex = np.mean([len(item.split(\" \")) for item in test_textsYandex])\n\nprint('Average length - Train:',meanLengthTrain,'Valid:',meanLengthValid,'TestGoogle:',meanLengthTestGoogle,'TestYandex:',meanLengthTestYandex)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d270d631-604a-4110-8b20-84f87577b9b5","_cell_guid":"8406af82-c60d-49ed-be44-aac7f425ad14","trusted":true},"cell_type":"code","source":"MAX_SENTENCE_LENGTH = int(meanLengthTrain + 20) # we let a text go 20 words longer than the mean text length (you can also tune this parameter).\n\n# Convert train, validation, and test text into lists with word ids\ntrainFeatures = tokenizer.texts_to_sequences(train_texts)\ntrainFeatures = pad_sequences(trainFeatures, MAX_SENTENCE_LENGTH, padding='post')\ntrainLabels = train_labels.values\n\nvalidFeatures = tokenizer.texts_to_sequences(valid_texts)\nvalidFeatures = pad_sequences(validFeatures, MAX_SENTENCE_LENGTH, padding='post')\nvalidLabels = valid_labels.values\n\ntestFeaturesGoogleLSTMwith = tokenizer.texts_to_sequences(test_textsGoogle)\ntestFeaturesGoogleLSTMwith = pad_sequences(testFeaturesGoogleLSTMwith, MAX_SENTENCE_LENGTH, padding='post')\n\ntestFeaturesYandexLSTMwith = tokenizer.texts_to_sequences(test_textsYandex)\ntestFeaturesYandexLSTMwith = pad_sequences(testFeaturesYandexLSTMwith, MAX_SENTENCE_LENGTH, padding='post')\ntrainFeatures","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9977029b-0089-4ac1-8a24-110e67b5698f","_cell_guid":"fafc8621-036f-4dcc-8b02-15ab2f99846e","trusted":true},"cell_type":"code","source":"#from: https://www.kaggle.com/bertcarremans/using-word-embeddings-for-sentiment-analysis/data\nEMBEDDING_FILE='../input/glove-twitter/glove.twitter.27B.25d.txt'\nemb_dict = {}\nglove = open(EMBEDDING_FILE)\nfor line in glove:\n    values = line.split()\n    word = values[0]\n    vector = np.asarray(values[1:], dtype='float32')\n    emb_dict[word] = vector\nglove.close()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"251246f7-1c41-4cd3-b424-e096fad9468d","_cell_guid":"748c1232-0250-4023-9278-28f059223b1c","trusted":true},"cell_type":"code","source":"#from: https://www.kaggle.com/bertcarremans/using-word-embeddings-for-sentiment-analysis/data\nembedding_matrix = np.zeros((VOCABULARY_SIZE, 25))\n\nfor w, i in tokenizer.word_index.items():\n    # The word_index contains a token for all words of the training data so we need to limit that\n    if i < VOCABULARY_SIZE:\n        vect = emb_dict.get(w)\n        # Check if the word from the training data occurs in the GloVe word embeddings\n        # Otherwise the vector is kept with only zeros\n        if vect is not None:\n            embedding_matrix[i] = vect\n    else:\n        break","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"366598c1-208e-4e50-876c-fb976f98e25b","_cell_guid":"722abe90-3de5-4186-990f-b37fc6a3dfd2","trusted":true},"cell_type":"code","source":"embedding_matrix.shape","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2a87b603-3e4f-4fd4-8145-3ed70bf31a88","_cell_guid":"a61e133a-7219-4d1e-adb5-97242dbde9e7","trusted":true},"cell_type":"code","source":"# Hyperparameters for model tuning\nLEARNING_RATE = 0.001\nBATCH_SIZE = 128\nEPOCHS = 9","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c6c314e4-7342-4c4f-a353-4f2c05523afb","_cell_guid":"2757465c-ad1f-4d42-99c7-768f769dc7c1","trusted":true},"cell_type":"code","source":"#LSTM\nLSTMwith = Sequential()\n\n# We use pre-trained embeddings from GloVe. These are fed in as a layer of our network and the weights do not update during the training process.\nLSTMwith.add(Embedding(input_dim=VOCABULARY_SIZE, output_dim=embedding_matrix.shape[1], weights=[embedding_matrix], trainable=False, input_length=len(trainFeatures[0])))\nLSTMwith.add(Bidirectional(LSTM(24)))\nLSTMwith.add(Dropout(0.6))\nLSTMwith.add(Dense(12, activation='relu'))\nLSTMwith.add(Dropout(0.5))\nLSTMwith.add(Dense(1, activation='sigmoid'))\n            \noptimizer = optimizers.Adam(lr=LEARNING_RATE)\nLSTMwith.compile(optimizer=optimizer,loss='binary_crossentropy',metrics=['accuracy'])\n\nprint(LSTMwith.summary())","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c51ea78a-6f20-43f0-8515-0686fd508fe1","_cell_guid":"216c2895-586e-4169-8fa9-1adad32eb55c","trusted":true},"cell_type":"code","source":"# ratio of non-toxic to toxic in training:\n200000/25000","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2c8e653f-9829-4862-b110-1f0a11ddfcce","_cell_guid":"192464dd-228f-428d-8338-eca4678cc346","trusted":true},"cell_type":"code","source":"# We have a class imbalance, upweight the toxic comments\nclass_weights = {0: 1,\n                 1: 8}","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"efef70c8-1016-44df-8322-61822bf4cc60","_cell_guid":"cb2079a1-f260-40aa-821b-63e66fae9ba1","trusted":true},"cell_type":"code","source":"roc = RocCallback(training_data=(trainFeatures, trainLabels),\n                  validation_data=(validFeatures, validLabels))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"4fe1246d-9bfb-402c-ac21-074bf4eb3139","_cell_guid":"295a885d-792a-4ec1-ad0f-eace1d6a24a9","trusted":true},"cell_type":"code","source":"# train model\nstart = time.time()    \nhistory = LSTMwith.fit(trainFeatures, trainLabels, validation_data = (validFeatures, validLabels), batch_size=BATCH_SIZE, epochs=EPOCHS, class_weight=class_weights, callbacks=[roc])\nprint(\"Training took %d seconds\" % (time.time() - start))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"c737aea0-4011-4e39-a30b-24589f46ea13","_cell_guid":"a890b438-21a7-4489-bc8f-e7c59de56405","trusted":true},"cell_type":"code","source":"pred_valid_LSTMwith = pd.DataFrame(LSTMwith.predict(validFeatures))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"2b52bf54-c3da-40cb-942a-64d5c3053397","_cell_guid":"c646d092-0181-4096-bbb0-f3b1bc189b2f","trusted":true},"cell_type":"code","source":"roc_auc_score(validLabels, pred_valid_LSTMwith)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"1f478ebd-5f7d-4b80-b39a-a879031bc6b8","_cell_guid":"205e4611-0205-46b2-a778-7a838b464c28","trusted":true},"cell_type":"code","source":"validPred = pred_valid_LSTMwith\npred_valid_binary = round(validPred)\n\nconfusionMatrix = None\nconfusionMatrix = confusion_matrix(validLabels, pred_valid_binary)\n\nplt.rcParams['figure.figsize'] = [3, 3] ## plot size\ndisplayConfusionMatrix(confusionMatrix)\nplt.title(\"Confusion Matrix\", fontsize=PLOT_FONT_SIZE)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f68b2b16-53dc-476a-890e-924b84b2cb0b","_cell_guid":"7447438f-9dcf-4f59-91ca-8b2ec0ef680d","trusted":true},"cell_type":"markdown","source":"# Make predictions"},{"metadata":{"_uuid":"cf317dfe-3551-4a4a-8c43-c89ee8db1090","_cell_guid":"c0e288e0-397e-4e65-98e6-ff1c9d6ed78f","trusted":true},"cell_type":"code","source":"# make test predictions (average both translations)\npredictionsGoogle = pd.DataFrame(LSTMwith.predict(testFeaturesGoogleLSTMwith))\npredictionsYandex = pd.DataFrame(LSTMwith.predict(testFeaturesYandexLSTMwith))\npredictions = (predictionsGoogle+predictionsYandex)/2\npredictions","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"72506800-46e8-4dad-a46c-47f1b385237d","_cell_guid":"fe26dd0f-6f0d-4b51-8ebd-dbf1df31d2fd","trusted":true},"cell_type":"code","source":"# prep for submission\nsample = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nsample['toxic'] = predictions\nsample","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"e9f557b6-72e0-45e8-aaaa-c1db8544e0be","_cell_guid":"7c3a426c-21c9-40fb-9dc4-14806e2b01bf","trusted":true},"cell_type":"code","source":"# make submission\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}