{"cells":[{"metadata":{},"cell_type":"markdown","source":"### Takeaway of DIfference Baselines\n\n(1) Multi-language training is more beneficial than using English only as a training. <br> \n(2) Vectorization is more better than Doc2Vec processing. Using the 'clean' data would also result in higher performance <br>\n(3) The performance of the different models ranked as followed: NB-SVM > xgboost > logistic regression ","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# You can write up to 5GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom scipy.sparse import hstack\nimport gc\nimport os\n\nfrom sklearn.model_selection import train_test_split\nfrom collections import Counter\n\nfrom gensim.models.phrases import Phraser,Phrases \nfrom gensim.models.word2vec import Word2Vec \nfrom gensim.models import Doc2Vec\nfrom gensim.models.doc2vec import TaggedDocument","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading data\n\ntrain_en = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain_en['lang'] = 'en'\n\ntrain_es = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')\ntrain_es['lang'] = 'es'\n\ntrain_fr = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv')\ntrain_fr['lang'] = 'fr'\n\ntrain_pt = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv')\ntrain_pt['lang'] = 'pt'\n\ntrain_ru = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv')\ntrain_ru['lang'] = 'ru'\n\ntrain_it = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv')\ntrain_it['lang'] = 'it'\n\ntrain_tr = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv')\ntrain_tr['lang'] = 'tr'\n\n#train2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n#train2.toxic = train2.toxic.round().astype(int)\n#train2['lang'] = 'en'\n\ndata = pd.concat([\n    \n    train_en[['comment_text', 'lang', 'toxic']],\n    train_es[['comment_text', 'lang', 'toxic']],\n    train_tr[['comment_text', 'lang', 'toxic']],\n    train_fr[['comment_text', 'lang', 'toxic']],\n    train_pt[['comment_text', 'lang', 'toxic']],\n    train_ru[['comment_text', 'lang', 'toxic']],\n    train_it[['comment_text', 'lang', 'toxic']]\n    \n]).sample(n=100000).reset_index(drop=True)\n\ndel train_en, train_es, train_fr, train_pt, train_ru, train_it, train_tr\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\nsubmission = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"validation.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.shape,validation.shape,submission.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['comment_text'][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['comment_text'][100]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lens = data.comment_text.str.len()\nlens.min(),lens.mean(), lens.std(), lens.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(data),len(validation)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['comment_text'].fillna(\"unknown\", inplace=True)\nvalidation['comment_text'].fillna(\"unknown\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re, string\nre_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')\ndef tokenize(s): return re_tok.sub(r' \\1 ', s).split()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#clean txt function \n#write a class function for cleaning and preprocss the doctoVec attributes \n#clean the text using genism \nfrom gensim import utils\nimport gensim.parsing.preprocessing as gsp\n\n#the filters are doing the following \n#remove tags \n#remove punctuation\n#standarized the spaces \n#stop words \n#stemming \n#lower case for all words \nfilters = [\n           gsp.strip_tags, \n           gsp.strip_punctuation,\n           gsp.strip_multiple_whitespaces,\n           gsp.strip_numeric,\n           gsp.remove_stopwords, \n           gsp.strip_short, \n           gsp.stem_text\n          ]\n\ndef clean_text(s):\n    s = str(s).lower()\n    s = utils.to_unicode(s)\n    for f in filters:\n        s = f(s)\n    return s","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = data.shape[0]\n\n#clean the text first \ndata[\"clean_comment\"] = data.comment_text.apply(clean_text)\nvalidation[\"clean_comment\"] = validation.comment_text.apply(clean_text)\nsubmission[\"clean_comment\"] = submission.content.apply(clean_text)\n\n#split the train and test from the whole table \nx_train, x_test, y_train, y_test = train_test_split(data[\"clean_comment\"],data.toxic, test_size=0.3)\n\n#vectorization of the model \nvec = TfidfVectorizer(ngram_range=(1,2), tokenizer=tokenize,\n               strip_accents='unicode', use_idf=1,\n               smooth_idf=1, sublinear_tf=1 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_term_doc = vec.fit_transform(x_train)\ntest_term_doc = vec.transform(x_test)\nvalid_term_doc = vec.transform(validation['clean_comment'])\nsubmission_term_doc = vec.transform(submission['clean_comment'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_term_doc.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.pipeline import Pipeline\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import cross_val_score\nimport xgboost as xgb\nfrom xgboost import XGBClassifier\nfrom sklearn.metrics import accuracy_score, f1_score\nfrom sklearn.metrics import roc_auc_score","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logreg = LogisticRegression(n_jobs=1, C=4)\nlogreg.fit(train_term_doc, y_train)\ny_pred = logreg.predict(test_term_doc)\n\nprint('Testing accuracy %s' % accuracy_score(y_test, y_pred))\nprint('Testing F1 score: {}'.format(f1_score(y_test, y_pred, average='weighted')))\nprint('Testing AUC score %s' % roc_auc_score(y_test, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#score the validation dataset\ny_valid = validation['toxic']\ny_pred_valid = logreg.predict(valid_term_doc)\nprint('Testing accuracy %s' % accuracy_score(y_valid, y_pred_valid))\nprint('Testing F1 score: {}'.format(f1_score(y_valid, y_pred_valid, average='weighted')))\nprint('Testing AUC score %s' % roc_auc_score(y_valid, y_pred_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#score the submission file \ny_pred_submission = logreg.predict(submission_term_doc)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#load the sample submission file \nsample_sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submid = pd.DataFrame({'id': sample_sub[\"id\"]})\nsubmission = pd.concat([submid, pd.DataFrame(y_pred_submission,columns=['toxic'])], axis=1)\n\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#run the xgboost \nxgboost_model = xgb.train({'max_depth': 5, \"seed\": 123,'objective':'binary:logistic',\n                   'learning_rate':0.23,'min_child_weight':4}, \n                  xgb.DMatrix(train_term_doc, label=y_train), num_boost_round=500)\n\ny_pred_xgboost = xgboost_model.predict(xgb.DMatrix(test_term_doc, label=y_test))\n\nprint('Testing accuracy %s' % accuracy_score(y_test, y_pred_xgboost.round()))\nprint('Testing F1 score: {}'.format(f1_score(y_test, y_pred_xgboost.round(), average='weighted')))\nprint('Testing AUC score %s' % roc_auc_score(y_test, y_pred_xgboost))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#validation set\ny_valid = validation['toxic']\ny_pred_valid = xgboost_model.predict(xgb.DMatrix(valid_term_doc, label=y_valid))\n#print('Testing accuracy %s' % accuracy_score(y_valid, y_pred_valid))\n#print('Testing F1 score: {}'.format(f1_score(y_valid, y_pred_valid, average='weighted')))\nprint('Testing AUC score %s' % roc_auc_score(y_valid, y_pred_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#score the submissint file \ny_pred_submission = xgboost_model.predict(xgb.DMatrix(submission_term_doc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nsubmid = pd.DataFrame({'id': sample_sub[\"id\"]})\nsubmission = pd.concat([submid, pd.DataFrame(y_pred_submission,columns=['toxic'])], axis=1)\n\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#use the English Version\ndata = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\nvalidation = pd.read_csv('../input/translated/validation_translate.csv')\nsubmission = pd.read_csv('../input/translated/test_translate.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = data.shape[0]\n\n#clean the text first \ndata[\"clean_comment\"] = data.comment_text.apply(clean_text)\nvalidation[\"clean_comment\"] = validation.comment_text.apply(clean_text)\nsubmission[\"clean_comment\"] = submission.content.apply(clean_text)\n\n#split the train and test from the whole table \nx_train, x_test, y_train, y_test = train_test_split(data[\"clean_comment\"],data.toxic, test_size=0.3)\n\n#vectorization of the model \nvec = TfidfVectorizer(ngram_range=(1,2), tokenizer=tokenize,\n               strip_accents='unicode', use_idf=1,\n               smooth_idf=1, sublinear_tf=1 )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_term_doc = vec.fit_transform(x_train)\ntest_term_doc = vec.transform(x_test)\nvalid_term_doc = vec.transform(validation['clean_comment'])\nsubmission_term_doc = vec.transform(submission['clean_comment'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"logreg = LogisticRegression(n_jobs=1, C=4)\nlogreg.fit(train_term_doc, y_train)\ny_pred = logreg.predict(test_term_doc)\n\nprint('Testing accuracy %s' % accuracy_score(y_test, y_pred))\nprint('Testing F1 score: {}'.format(f1_score(y_test, y_pred, average='weighted')))\nprint('Testing AUC score %s' % roc_auc_score(y_test, y_pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#score the validation dataset\ny_valid = validation['toxic']\ny_pred_valid = logreg.predict(valid_term_doc)\nprint('Testing accuracy %s' % accuracy_score(y_valid, y_pred_valid))\nprint('Testing F1 score: {}'.format(f1_score(y_valid, y_pred_valid, average='weighted')))\nprint('Testing AUC score %s' % roc_auc_score(y_valid, y_pred_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nsubmid = pd.DataFrame({'id': sample_sub[\"id\"]})\nsubmission = pd.concat([submid, pd.DataFrame(y_pred_submission,columns=['toxic'])], axis=1)\n\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train the xgboost model \n#run the xgboost \nxgboost_model = xgb.train({'max_depth': 5, \"seed\": 123,'objective':'binary:logistic',\n                   'learning_rate':0.23,'min_child_weight':4}, \n                  xgb.DMatrix(train_term_doc, label=y_train), num_boost_round=500)\n\ny_pred_xgboost = xgboost_model.predict(xgb.DMatrix(test_term_doc, label=y_test))\n\nprint('Testing accuracy %s' % accuracy_score(y_test, y_pred_xgboost.round()))\nprint('Testing F1 score: {}'.format(f1_score(y_test, y_pred_xgboost.round(), average='weighted')))\nprint('Testing AUC score %s' % roc_auc_score(y_test, y_pred_xgboost))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#validation set\ny_valid = validation['toxic']\ny_pred_valid = xgboost_model.predict(xgb.DMatrix(valid_term_doc, label=y_valid))\n#print('Testing accuracy %s' % accuracy_score(y_valid, y_pred_valid))\n#print('Testing F1 score: {}'.format(f1_score(y_valid, y_pred_valid, average='weighted')))\nprint('Testing AUC score %s' % roc_auc_score(y_valid, y_pred_valid))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#score the submissint file \ny_pred_submission = xgboost_model.predict(xgb.DMatrix(submission_term_doc))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sample_sub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\nsubmid = pd.DataFrame({'id': sample_sub[\"id\"]})\nsubmission = pd.concat([submid, pd.DataFrame(y_pred_submission,columns=['toxic'])], axis=1)\n\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Naive-Bayes Code Reference \nhttps://medium.com/towards-artificial-intelligence/naive-bayes-support-vector-machine-svm-art-of-state-results-hands-on-guide-using-fast-ai-13b5d9bea3b2","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#Navie-Bayes-SVM version- use multi-language\n# Loading data\n\ntrain_en = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain_en['lang'] = 'en'\n\ntrain_es = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')\ntrain_es['lang'] = 'es'\n\ntrain_fr = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv')\ntrain_fr['lang'] = 'fr'\n\ntrain_pt = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv')\ntrain_pt['lang'] = 'pt'\n\ntrain_ru = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv')\ntrain_ru['lang'] = 'ru'\n\ntrain_it = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv')\ntrain_it['lang'] = 'it'\n\ntrain_tr = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv')\ntrain_tr['lang'] = 'tr'\n\n#train2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n#train2.toxic = train2.toxic.round().astype(int)\n#train2['lang'] = 'en'\n\ntrain = pd.concat([\n    \n    train_en[['comment_text', 'lang', 'toxic']],\n    train_es[['comment_text', 'lang', 'toxic']],\n    train_tr[['comment_text', 'lang', 'toxic']],\n    train_fr[['comment_text', 'lang', 'toxic']],\n    train_pt[['comment_text', 'lang', 'toxic']],\n    train_ru[['comment_text', 'lang', 'toxic']],\n    train_it[['comment_text', 'lang', 'toxic']]\n    \n]).sample(n=500000).reset_index(drop=True)\n\ndel train_en, train_es, train_fr, train_pt, train_ru, train_it, train_tr\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub= pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_cols = ['toxic']\ntrain['none'] = 1-train[label_cols].max(axis=1)\ntrain.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = train.shape[0]\n\nvec = TfidfVectorizer(ngram_range=(1,2), tokenizer=tokenize,\n               strip_accents='unicode', use_idf=1,\n               smooth_idf=1, sublinear_tf=1 )\n\n\ntrn_term_doc = vec.fit_transform(train['comment_text'])\ntest_term_doc = vec.transform(test['content'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pr(y_i, y):\n    p = x[y==y_i].sum(0)\n    return (p+1) / ((y==y_i).sum()+1)\n\nx = trn_term_doc\ntest_x = test_term_doc\n\ndef get_mdl(y):\n    y = y.values\n    r = np.log(pr(1,y) / pr(0,y))\n    m = LogisticRegression(C=4, dual=False)\n    x_nb = x.multiply(r)\n    return m.fit(x_nb, y), r\n\npreds = np.zeros((len(test), len(label_cols)))\n\nfor i, j in enumerate(label_cols):\n    m,r = get_mdl(train[j])\n    preds[:,i] = m.predict_proba(test_x.multiply(r))[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submid = pd.DataFrame({'id': sub[\"id\"]})\nsubmission = pd.concat([submid, pd.DataFrame(preds, columns = label_cols)], axis=1)\n\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}