{"cells":[{"metadata":{},"cell_type":"markdown","source":"Word CNN without pre-trained embeddings. Only using one of the training sets. Relies on translated test and validation data from @bamps53 and @kashnitsky "},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# # Load packages\n\n# Ignore warnings\nimport warnings\n\ndef warn(*args, **kwargs):\n    pass\n\nwarnings.warn = warn\n\nimport os\nimport numpy as np\nimport pandas as pd\n\nimport keras\nfrom keras import *\nfrom keras import layers\nfrom keras.layers import Embedding, Conv1D, GlobalMaxPooling1D, Dense, Dropout\nfrom keras.models import Model\nfrom keras.preprocessing import *\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\nfrom keras.callbacks import Callback\n\nfrom sklearn.metrics import accuracy_score\nfrom sklearn.metrics import roc_auc_score, confusion_matrix\n\nimport seaborn as sns\nimport matplotlib.pyplot as plt\n\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.stem import WordNetLemmatizer\nimport re\nfrom io import StringIO","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Helper functions"},{"metadata":{"trusted":true},"cell_type":"code","source":"class RocCallback(Callback):\n    def __init__(self,training_data,validation_data):\n        self.x = training_data[0]\n        self.y = training_data[1]\n        self.x_val = validation_data[0]\n        self.y_val = validation_data[1]\n    \n    def on_train_begin(self, logs={}):\n        return\n\n    def on_train_end(self, logs={}):\n        return\n\n    def on_epoch_begin(self, epoch, logs={}):\n        return\n\n    def on_epoch_end(self, epoch, logs={}):\n        y_pred_train = self.model.predict_proba(self.x)\n        roc_train = roc_auc_score(self.y, y_pred_train)\n        y_pred_val = self.model.predict_proba(self.x_val)\n        roc_val = roc_auc_score(self.y_val, y_pred_val)\n        print('\\rroc-auc_train: %s - roc-auc_val: %s' % (str(round(roc_train,4)),str(round(roc_val,4))),end=100*' '+'\\n')\n        return\n\n    def on_batch_begin(self, batch, logs={}):\n        return\n\n    def on_batch_end(self, batch, logs={}):\n        return\n    \n\ndef genAUC(validFeatures,validLabels):\n    pred_valid_df = pd.DataFrame(model.predict(validFeatures))   \n    auc = roc_auc_score(validLabels, pred_valid_df)\n    print('AUC: %.3f' % auc)\n\n    \nPLOT_FONT_SIZE = 10    #font size for axis of plots\n\n#define helper function for confusion matrix\n\ndef displayConfusionMatrix(confusionMatrix):\n    \"\"\"Confusion matrix plot\"\"\"\n    \n    confusionMatrix = np.transpose(confusionMatrix)\n    \n    ## calculate class level precision and recall from confusion matrix\n    precisionLow = round((confusionMatrix[0][0] / (confusionMatrix[0][0] + confusionMatrix[0][1]))*100, 1)\n    precisionHigh = round((confusionMatrix[1][1] / (confusionMatrix[1][0] + confusionMatrix[1][1]))*100, 1)\n    recallLow = round((confusionMatrix[0][0] / (confusionMatrix[0][0] + confusionMatrix[1][0]))*100, 1)\n    recallHigh = round((confusionMatrix[1][1] / (confusionMatrix[0][1] + confusionMatrix[1][1]))*100, 1)\n\n    ## show heatmap\n    plt.imshow(confusionMatrix, interpolation='nearest',cmap=plt.cm.Blues,vmin=0, vmax=100)\n    \n    ## axis labeling\n    xticks = np.array([0,1])\n    plt.gca().set_xticks(xticks)\n    plt.gca().set_yticks(xticks)\n    plt.gca().set_xticklabels([\"Not Toxic \\n Recall=\" + str(recallLow), \"Toxic \\n Recall=\" + str(recallHigh)], fontsize=PLOT_FONT_SIZE)\n    plt.gca().set_yticklabels([\"Not Toxic \\n Precision=\" + str(precisionLow), \"Toxic \\n Precision=\" + str(precisionHigh)], fontsize=PLOT_FONT_SIZE)\n    plt.ylabel(\"Predicted Class\", fontsize=PLOT_FONT_SIZE)\n    plt.xlabel(\"Actual Class\", fontsize=PLOT_FONT_SIZE)\n        \n    ## add text in heatmap boxes\n    addText(xticks, xticks, confusionMatrix)\n    \ndef addText(xticks, yticks, results):\n    \"\"\"Add text in the plot\"\"\"\n    for i in range(len(yticks)):\n        for j in range(len(xticks)):\n            text = plt.text(j, i, results[i][j], ha=\"center\", va=\"center\", color=\"white\", size=PLOT_FONT_SIZE) ### size here is the size of text inside a single box in the heatmap\n\n            \ndef plotConfusion(validFeatures,validLabels):\n    \n    pred_valid_df = pd.DataFrame(model.predict(validFeatures))\n    pred_valid_binary = round(pred_valid_df)\n\n    confusionMatrix = None\n    confusionMatrix = confusion_matrix(validLabels, pred_valid_binary)\n\n    plt.rcParams['figure.figsize'] = [3, 3] ## plot size\n    displayConfusionMatrix(confusionMatrix)\n    plt.title(\"Confusion Matrix\", fontsize=PLOT_FONT_SIZE)\n    plt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def lemmetize_data(data,field):\n    cleaned_texts = []\n    for text in data[field]: # Loop through the tokens (the words or symbols) \n        cleaned_text = text.lower()  # Convert the text to lower case\n        cleaned_text = ' '.join([word for word in cleaned_text.split() if word not in stopset])  # Keep only words that are not stopwords.\n        cleaned_text = ' '.join([wordnet_lemmatizer.lemmatize(word, pos='n') for word in cleaned_text.split()])  # Keep each noun's lemma.\n        cleaned_text = ' '.join([wordnet_lemmatizer.lemmatize(word, pos='v') for word in cleaned_text.split()])  # Keep each verb's lemma.\n        cleaned_text = re.sub(r\"(http\\S+)\",\" \", cleaned_text)  # Remove http links.\n        cleaned_text = re.sub(\"[^a-zA-Z]\",\" \", cleaned_text)  # Remove numbers and punctuation.\n        cleaned_text = ' '.join(cleaned_text.split())  # Remove white space.\n        cleaned_texts.append(cleaned_text) \n    data['cleanText'] = cleaned_texts","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nltk.download('stopwords')\nnltk.download('wordnet')\n\nwordnet_lemmatizer = WordNetLemmatizer()\nstopset = list(set(stopwords.words('english')))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Load data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# load training data 1\ntrain_comment=pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\ntrain_texts = train_comment['comment_text']\n\n#lemmetize_data(train_comment,'comment_text')\n#train_texts = train_comment['cleanText']\n\ntrain_labels = train_comment['toxic']\ntrain_comment","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load validation data\nvalid=pd.read_csv('../input/val-en-df/validation_en.csv')\nvalid_texts = valid['comment_text_en']\n\n#lemmetize_data(valid,'comment_text_en')\n#valid_texts = valid['cleanText']\n\nvalid_labels = valid['toxic']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load testing data (Translated via Google)\ntest_google=pd.read_csv('../input/test-en-df/test_en.csv')\ntest_textsGoogle = test_google['content_en']\n\n#lemmetize_data(test_google,'content_en')\n#test_textsGoogle = test_google['cleanText']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# load testing data (Translated via Yandex)\ntest_yandex=pd.read_csv('../input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')\ntest_textsYandex = test_yandex['translated']\n\n#lemmetize_data(test_yandex,'translated')\n#test_textsYandex = test_yandex['cleanText']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def plotCases(data):\n    cases_count = data.value_counts(dropna=False)\n\n    # Plot  results \n    plt.figure(figsize=(6,6))\n    sns.barplot(x=cases_count.index, y=cases_count.values)\n    plt.ylabel('Texts', fontsize=12)\n    plt.xticks(range(len(cases_count.index)), ['Not', 'Toxic'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotCases(train_labels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotCases(valid_labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Prepare data"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define vocabulary size (you can tune this parameter and evaluate model performance)\nVOCABULARY_SIZE = 5000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Create input feature arrays\ntokenizer = Tokenizer(num_words=VOCABULARY_SIZE)\ntokenizer.fit_on_texts(train_texts)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Convert words into word ids\nmeanLengthTrain = np.mean([len(item.split(\" \")) for item in train_texts])\nmeanLengthValid = np.mean([len(item.split(\" \")) for item in valid_texts])\nmeanLengthTestGoogle = np.mean([len(item.split(\" \")) for item in test_textsGoogle])\nmeanLengthTestYandex = np.mean([len(item.split(\" \")) for item in test_textsYandex])\n\nprint('Average length - Train:',meanLengthTrain,'Valid:',meanLengthValid,'TestGoogle:',meanLengthTestGoogle,'TestYandex:',meanLengthTestYandex)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"MAX_SENTENCE_LENGTH = int(meanLengthTrain + 10) # we let a text go 10 words longer than the mean text length (you can also tune this parameter).\n\n# Convert train, validation, and test text into lists with word ids\ntrainFeatures = tokenizer.texts_to_sequences(train_texts)\ntrainFeatures = pad_sequences(trainFeatures, MAX_SENTENCE_LENGTH, padding='post')\ntrainLabels = train_labels.values\n\nvalidFeatures = tokenizer.texts_to_sequences(valid_texts)\nvalidFeatures = pad_sequences(validFeatures, MAX_SENTENCE_LENGTH, padding='post')\nvalidLabels = valid_labels.values\n\ntestFeaturesGoogle = tokenizer.texts_to_sequences(test_textsGoogle)\ntestFeaturesGoogle = pad_sequences(testFeaturesGoogle, MAX_SENTENCE_LENGTH, padding='post')\n\ntestFeaturesYandex = tokenizer.texts_to_sequences(test_textsYandex)\ntestFeaturesYandex = pad_sequences(testFeaturesYandex, MAX_SENTENCE_LENGTH, padding='post')","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train Word CNN without pre-trained embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Define filter and kernel size for CNN (can adjust in tuning model)\nFILTERS_SIZE = 16\nKERNEL_SIZE = 5","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Overfits very quickly, use super low learning rate\n\n# Define embeddings dimensions (columns in matrix fed into CNN and nodes in hidden layer of built-in keras function)\nEMBEDDINGS_DIM = 20\n\n# Hyperparameters for model tuning\nLEARNING_RATE = 0.0001\nBATCH_SIZE = 500\nEPOCHS = 8","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Word CNN\nmodel = Sequential()\n\n# We use built-in keras funtion to generate embeddings. Another option is pre-trained embeddings with Word2vec or GloVe.\nmodel.add(Embedding(input_dim=VOCABULARY_SIZE + 1, output_dim=EMBEDDINGS_DIM, input_length=len(trainFeatures[0])))\nmodel.add(Conv1D(FILTERS_SIZE, KERNEL_SIZE, activation='relu'))\nmodel.add(Dropout(0.5))\nmodel.add(GlobalMaxPooling1D())\nmodel.add(Dropout(0.5))\nmodel.add(Dense(12, activation='relu'))\nmodel.add(Dense(1, activation='sigmoid'))\n            \noptimizer = optimizers.Adam(lr=LEARNING_RATE)\nmodel.compile(optimizer=optimizer,loss='binary_crossentropy',metrics=['accuracy'])\n\nprint(model.summary())","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# ratio of non-toxic to toxic in training:\n200000/25000","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# We have a class imbalance, upweight the toxic comments\nclass_weights = {0: 1,\n                 1: 8}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc = RocCallback(training_data=(trainFeatures, trainLabels),\n                  validation_data=(validFeatures, validLabels))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train model\nhistory = model.fit(trainFeatures, trainLabels, validation_data = (validFeatures, validLabels), batch_size=BATCH_SIZE, epochs=EPOCHS, class_weight=class_weights, callbacks=[roc])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"genAUC(validFeatures,validLabels)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plotConfusion(validFeatures,validLabels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Make test predictions"},{"metadata":{"trusted":true},"cell_type":"code","source":"# make test predictions (average both translations)\npredictionsGoogle = pd.DataFrame(model.predict(testFeaturesGoogle))\npredictionsYandex = pd.DataFrame(model.predict(testFeaturesYandex))\npredictions = (predictionsGoogle+predictionsYandex)/2\npredictions","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# prep for submission\nsample = pd.read_csv(\"../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\nsample['toxic'] = predictions[0]\nsample","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# make submission\nsample.to_csv(\"submission.csv\", index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}