{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import os, time\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom kaggle_datasets import KaggleDatasets\nimport sys\nfrom time import time\nimport matplotlib.pyplot as plt\nimport pandas as pd\nimport numpy as np\nfrom sklearn.utils import resample\nfrom sklearn.feature_extraction.text import TfidfVectorizer\nfrom sklearn.feature_extraction.text import HashingVectorizer\nfrom sklearn.feature_selection import SelectFromModel\nfrom sklearn.feature_selection import SelectKBest, chi2\nfrom sklearn.linear_model import RidgeClassifier\nfrom sklearn.pipeline import Pipeline\nfrom sklearn.svm import LinearSVC, SVC\nfrom sklearn.linear_model import SGDClassifier\nfrom sklearn.linear_model import Perceptron\nfrom sklearn.linear_model import PassiveAggressiveClassifier\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.naive_bayes import BernoulliNB, ComplementNB, MultinomialNB, GaussianNB\nfrom sklearn.model_selection import GridSearchCV\nfrom sklearn.metrics import classification_report\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.neighbors import NearestCentroid\nfrom sklearn.ensemble import RandomForestClassifier\nfrom sklearn.utils.extmath import density\nfrom sklearn import metrics\nfrom sklearn.model_selection import train_test_split\n# We'll use a tokenizer for the BERT model from the modelling demo notebook.\n!pip install bert-tensorflow\nimport bert.tokenization\n\nprint(tf.version.VERSION)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"SEQUENCE_LENGTH = 128\n\nDATA_PATH =  \"../input/jigsaw-multilingual-toxic-comment-classification\"\nBERT_PATH = \"../input/bert-multi\"\nBERT_PATH_SAVEDMODEL = os.path.join(BERT_PATH, \"bert_multi_from_tfhub\")\n\nOUTPUT_PATH = \"/kaggle/working\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Examples\n\nLoad and look at examples from [our first competition](https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge/). These are comments from Wikipedia with a variety of annotations (toxic, obscene, threat, etc).","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Training data from our first competition,\n# https://www.kaggle.com/c/jigsaw-toxic-comment-classification-challenge/data\nwiki_toxic_comment_data = \"jigsaw-toxic-comment-train.csv\"\nwiki_toxic_comment_data = \"jigsaw-toxic-comment-train-processed-seqlen128.csv\"\n\nwiki_toxic_comment_train = pd.read_csv(os.path.join(\n    DATA_PATH, wiki_toxic_comment_data))\nwiki_toxic_comment_train.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Generating Balance Data","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"wiki_toxic_comment_train['comment_text'][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic = wiki_toxic_comment_train[wiki_toxic_comment_train.toxic ==1]\nnot_toxic = wiki_toxic_comment_train[wiki_toxic_comment_train.toxic == 0]\n\ndownsampled = resample(not_toxic,\n                       replace = False, # sample without replacement\n                       n_samples = len(toxic), # match minority n\n                       random_state = 10) # reproducible results\ntrain = pd.concat([downsampled, toxic])\ntrain.toxic.value_counts().plot(kind = 'bar')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vectorizer = HashingVectorizer(decode_error='ignore', n_features=2 ** 18,\n                               alternate_sign=False)\n\nX = train['comment_text']\ny = train['toxic']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train_text, X_test_text, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=2)\nX_train = vectorizer.transform(X_train_text)\nX_test = vectorizer.transform(X_test_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def benchmark(clf, name):\n    print('_' * 80)\n    print(\"Training: \")\n    print(clf)\n    t0 = time()\n    clf.fit(X_train, y_train)\n    train_time = time() - t0\n    print(\"train time: %0.3fs\" % train_time)\n\n    t0 = time()\n    pred = clf.predict(X_test)\n    test_time = time() - t0\n    print(\"test time:  %0.3fs\" % test_time)\n\n    score = metrics.accuracy_score(y_test, pred)\n    print(\"accuracy:   %0.3f\" % score)\n    \n    auc = metrics.roc_auc_score(y_test, pred)\n    print(\"auc:      %0.3f\" % auc)\n    \n    if hasattr(clf, 'coef_'):\n        print(\"dimensionality: %d\" % clf.coef_.shape[1])\n        print(\"density: %f\" % density(clf.coef_))\n\n        print(\"classification report:\")\n        print(metrics.classification_report(y_test, pred,\n                                           ))\n\n        print(\"confusion matrix:\")\n        print(metrics.confusion_matrix(y_test, pred))\n\n    print()\n    clf_descr = name\n    return clf_descr, auc, train_time, test_time","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"results = []\nfor clf, name in (\n        (LogisticRegression(C = 1), \"Logistic Regression C = 1\"),\n        (LogisticRegression(C = 10), \"Logistic Regression C = 10\")):\n#        (LogisticRegression(C = 100), \"Logistic Regression C = 100\")):\n#        (RidgeClassifier, \"Ridge Classifier\"),\n#        (KNeighborsClassifier(n_neighbors=10), \"kNN\"),\n#        (RandomForestClassifier(), \"Random forest\")):\n    print('=' * 80)\n    print(name)\n    results.append(benchmark(clf,name))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for clf, name in (\n        (RandomForestClassifier(n_estimators = 10), \"Random forest tree=10\"),\n#       (RandomForestClassifier(n_estimators = 100), \"Random forest tree=100\"),\n        (RandomForestClassifier(n_estimators = 1000), \"Random forest tree=100\")):\n    print('=' * 80)\n    print(name)\n    results.append(benchmark(clf,name))\n    \ntuned_parameters = {'n_estimators':[10,100,1000],'max_features':[\"auto\",\"log2\"],'criterion': ['gini', 'entropy']}","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"for clf, name in (\n        (RandomForestClassifier(criterion = 'entropy'), \"Random forest criterion = entropy\"),\n        (RandomForestClassifier(max_features = 'log2'), \"Random forest max_features = log2\")):\n    print('=' * 80)\n    print(name)\n    results.append(benchmark(clf,name))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for clf, name in (\n    (LinearSVC(penalty='l2',  dual=False, tol=1e-3), \"LinearSVC L2\"),\n    (LinearSVC(penalty = 'l1',C = 0.01, dual=False, tol=1e-3), \"LinearSVC L1 C = 0.01\"),\n    (LinearSVC(penalty = 'l1',C = 0.1, dual=False, tol=1e-3), \"LinearSVC L1 C = 0.1\"),    \n    (LinearSVC(penalty = 'l1',C = 10, dual=False, tol=1e-3), \"LinearSVC L1 C = 10\"),\n#    (SVC(kernel = 'poly'), \"SVC Poly Kernel\"),\n (SVC(kernel = 'rbf'),\"SVC RBF Kernel\" )):\n    print('=' * 80)\n    print(name)\n    results.append(benchmark(clf,name))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Train sparse Naive Bayes classifiers\nprint(\"Naive Bayes\")\nfor alpha in [0.001, 0.01, 0.1, 1,10]:\n    results.append(benchmark(MultinomialNB(alpha=alpha), \"MultinomialNB Alpha=\"+str(alpha)))\n    results.append(benchmark(BernoulliNB(alpha=alpha), \"BernoulliNB Alpha=\"+str(alpha)))\n    results.append(benchmark(ComplementNB(alpha=alpha), \"ComplementNB Alpha=\"+str(alpha)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Hyperparameter Tuning","execution_count":null},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"\n# Set the parameters by cross-validation\n#LogisticRegression\ntuned_parameters = {'C': [0.1,1,10,100]}\n\nclf = GridSearchCV(LogisticRegression(), tuned_parameters, scoring= 'accuracy')\nclf.fit(X_train, y_train)\nprint(\"Best parameters set found on development set:\")\nprint(clf.best_params_)\nprint(\"Grid scores on development set:\")\nmeans = clf.cv_results_['mean_test_score']\nstds = clf.cv_results_['std_test_score']\nfor mean, std, params in zip(means, stds, clf.cv_results_['params']):\n    print(\"%0.3f (+/-%0.03f) for %r\"\n          % (mean, std * 2, params))\ny_true, y_pred = y_test, clf.predict(X_test)\nprint(classification_report(y_true, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"#RandomForestClassifier\n# Set the parameters by cross-validation\ntuned_parameters = {'n_estimators':[10,100,1000],'max_features':[\"auto\",\"log2\"],'criterion': ['gini', 'entropy']}\nclf = GridSearchCV(RandomForestClassifier(), tuned_parameters, scoring= 'accuracy')\nclf.fit(X_train, y_train)\nprint(\"Best parameters set found on development set:\")\nprint(clf.best_params_)\nprint(\"Grid scores on development set:\")\nmeans = clf.cv_results_['mean_test_score']\nstds = clf.cv_results_['std_test_score']\nfor mean, std, params in zip(means, stds, clf.cv_results_['params']):\n    print(\"%0.3f (+/-%0.03f) for %r\"\n          % (mean, std * 2, params))\ny_true, y_pred = y_test, clf.predict(X_test)\nprint(classification_report(y_true, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#SVM\ntuned_parameters = [{'kernel': ['rbf','poly','sigmoid'], 'C': [1, 10, 100, 1000]},\n                    {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]\n\nclf = GridSearchCV(SVC(), tuned_parameters, scoring= 'accuracy')\nclf.fit(X_train, y_train)\nprint(\"Best parameters set found on development set:\")\nprint(clf.best_params_)\nprint(\"Grid scores on development set:\")\nmeans = clf.cv_results_['mean_test_score']\nstds = clf.cv_results_['std_test_score']\nfor mean, std, params in zip(means, stds, clf.cv_results_['params']):\n    print(\"%0.3f (+/-%0.03f) for %r\"\n          % (mean, std * 2, params))\ny_true, y_pred = y_test, clf.predict(X_test)\nprint(classification_report(y_true, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"res_df = pd.DataFrame(results, columns = [\"Classifer\", \"AUC\", \"Train Time\", \"Test Time\"])\nres_df = res_df.sort_values(\"AUC\", ascending = False)\nres_df","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"indices = np.arange(len(results))\nres = [[x[i] for x in results] for i in range(4)]\nclf_names, score, training_time, test_time = res\ntraining_time = np.array(training_time) / np.max(training_time)\ntest_time = np.array(test_time) / np.max(test_time)\n\nplt.figure(figsize=(12, 8))\nplt.title(\"Score\")\nplt.barh(indices, score, .2, label=\"score\", color='navy')\nplt.barh(indices + .3, training_time, .2, label=\"training time\",\n         color='c')\nplt.barh(indices + .6, test_time, .2, label=\"test time\", color='darkorange')\nplt.yticks(())\nplt.legend(loc='best')\nplt.subplots_adjust(left=.25)\nplt.subplots_adjust(top=.95)\nplt.subplots_adjust(bottom=.05)\n\nfor i, c in zip(indices, clf_names):\n    plt.text(-.3, i, c)\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Ensemble","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Voting","execution_count":null},{"metadata":{"trusted":true,"collapsed":true},"cell_type":"code","source":"clf1 = LinearSVC(penalty='l1', dual=False, tol=1e-3)\nclf2 = LogisticRegression(C= 10)\nclf3 = ComplementNB(alpha=0.01)\nclf4 = RandomForestClassifier(n_estimators = 10)\nclf5 = MultinomialNB(alpha=0.01)\n\nclf1.fit(X_train, y_train)\npred1 = clf1.predict(X_test)\n\nclf2.fit(X_train, y_train)\npred2 = clf2.predict(X_test)\n\nclf3.fit(X_train, y_train)\npred3 = clf3.predict(X_test)\n\nclf4.fit(X_train, y_train)\npred4 = clf4.predict(X_test)\n\nclf5.fit(X_train, y_train)\npred5 = clf5.predict(X_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = (pred1 + pred2 + pred3 + pred4 + pred5)/5\nauc = metrics.roc_auc_score(y_test, pred)\nprint(\"auc:      %0.3f\" % auc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred1_prob = pred1\npred2_prob = [x[1] for x in clf2.predict_proba(X_test)]\npred3_prob = [x[1] for x in clf3.predict_proba(X_test)]\npred4_prob = [x[1] for x in clf4.predict_proba(X_test)]\npred5_prob = [x[1] for x in clf5.predict_proba(X_test)]\n\npred_prob = (pred1_prob + pred2_prob + pred3_prob + pred4_prob + pred5_prob)/5\nauc = metrics.roc_auc_score(y_test, pred_prob)\nprint(\"auc:      %0.3f\" % auc)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Boosting","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Top useful feature","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer\n\nvectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,stop_words='english')\nX_train_text, X_test_text, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=2)\nX_train = vectorizer.fit_transform(X_train_text)\nX_test = vectorizer.transform(X_test_text)\nfeature_names = vectorizer.get_feature_names()\nfeature_names = np.asarray(feature_names)\nfeature_names","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"clf1 = LogisticRegression()\nclf1.fit(X_train,y_train)\ntop100 = np.argsort(clf1.coef_)[-100:]\npd.DataFrame(feature_names[top100]).transpose()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# GloVe","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Plot","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# BERT Tokenizer\n\nGet the tokenizer corresponding to our multilingual BERT model. See [TensorFlow \nHub](https://tfhub.dev/tensorflow/bert_multi_cased_L-12_H-768_A-12/1) for more information about the model.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_tokenizer(bert_path=BERT_PATH_SAVEDMODEL):\n    \"\"\"Get the tokenizer for a BERT layer.\"\"\"\n    bert_layer = tf.saved_model.load(bert_path)\n    bert_layer = hub.KerasLayer(bert_layer, trainable=False)\n    vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()\n    cased = bert_layer.resolved_object.do_lower_case.numpy()\n    tf.gfile = tf.io.gfile  # for bert.tokenization.load_vocab in tokenizer\n    tokenizer = bert.tokenization.FullTokenizer(vocab_file, cased)\n  \n    return tokenizer\n\ntokenizer = get_tokenizer()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can look at one of our example sentences after we tokenize it, and then again after converting it to word IDs for BERT.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"example_sentence = wiki_toxic_comment_train.iloc[1].comment_text[:150]\nprint(example_sentence)\n\nexample_tokens = tokenizer.tokenize(example_sentence)\nprint(example_tokens[:17])\n\nexample_input_ids = tokenizer.convert_tokens_to_ids(example_tokens)\nprint(example_input_ids[:17])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Preprocessing\n\nProcess individual sentences for input to BERT using the tokenizer, and then prepare the entire dataset. The same code will process the other training data files, as well as the validation and test data.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def process_sentence(sentence, max_seq_length=SEQUENCE_LENGTH, tokenizer=tokenizer):\n    \"\"\"Helper function to prepare data for BERT. Converts sentence input examples\n    into the form ['input_word_ids', 'input_mask', 'segment_ids'].\"\"\"\n    # Tokenize, and truncate to max_seq_length if necessary.\n    tokens = tokenizer.tokenize(sentence)\n    if len(tokens) > max_seq_length - 2:\n        tokens = tokens[:(max_seq_length - 2)]\n\n    # Convert the tokens in the sentence to word IDs.\n    input_ids = tokenizer.convert_tokens_to_ids([\"[CLS]\"] + tokens + [\"[SEP]\"])\n\n    # The mask has 1 for real tokens and 0 for padding tokens. Only real\n    # tokens are attended to.\n    input_mask = [1] * len(input_ids)\n\n    # Zero-pad up to the sequence length.\n    pad_length = max_seq_length - len(input_ids)\n    input_ids.extend([0] * pad_length)\n    input_mask.extend([0] * pad_length)\n\n    # We only have one input segment.\n    segment_ids = [0] * max_seq_length\n\n    return (input_ids, input_mask, segment_ids)\n\ndef preprocess_and_save_dataset(unprocessed_filename, text_label='comment_text',\n                                seq_length=SEQUENCE_LENGTH, verbose=True):\n    \"\"\"Preprocess a CSV to the expected TF Dataset form for multilingual BERT,\n    and save the result.\"\"\"\n    dataframe = pandas.read_csv(os.path.join(DATA_PATH, unprocessed_filename),\n                                index_col='id')\n    processed_filename = (unprocessed_filename.rstrip('.csv') +\n                          \"-processed-seqlen{}.csv\".format(SEQUENCE_LENGTH))\n\n    pos = 0\n    start = time.time()\n\n    while pos < len(dataframe):\n        processed_df = dataframe[pos:pos + 10000].copy()\n\n        processed_df['input_word_ids'], processed_df['input_mask'], processed_df['all_segment_id'] = (\n            zip(*processed_df[text_label].apply(process_sentence)))\n\n        if pos == 0:\n            processed_df.to_csv(processed_filename, index_label='id', mode='w')\n        else:\n            processed_df.to_csv(processed_filename, index_label='id', mode='a',\n                                header=False)\n\n        if verbose:\n            print('Processed {} examples in {}'.format(\n                pos + 10000, time.time() - start))\n        pos += 10000\n    return\n  \n# Process the training dataset.\npreprocess_and_save_dataset(wiki_toxic_comment_data)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}