{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Multilingual NB-SVM with separate validation set presented / non-presented languages processing\n### log ensembling is using.\n#### Upvote please","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"from sklearn import *\nimport numpy as np\nimport pandas as pd\nimport glob\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score\n\ntrain = pd.read_csv('../input/jigsaw-train-translated-yandex-api/train_yandex.csv')\nval = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/validation.csv', usecols=['comment_text', 'toxic', 'lang'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/test.csv')\ntest['comment_text'] = test['content']\ntest['toxic'] = 0.5","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We have two different datasets. The first is a train which contains 6 laguages (after translation). Many toxic words are lost while translation. \nThe second is a validation dataset which contains 3 languages","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"val_languages = val.lang.unique().tolist()\nval_languages","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"non_val_languages = [l for l in test.lang.unique() if l not in val_languages]\nnon_val_languages","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_non_val_lang = train[train.lang.isin(non_val_languages)].sample(frac = 1).reset_index()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_non_val_lang.head(10)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Train dataset can be resampled with equal numbers of toxic and non-toxic comments","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic_count = train.toxic.sum()\nprint(toxic_count)\ntrain = pd.concat([train[train.toxic == 1], train[train.toxic == 0].sample(toxic_count + 5000)]).sample(frac = 1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Validation set can be splitted into an additional dataset and a small validation dataset.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"[train_v, val] = train_test_split(val, test_size = 0.05, random_state = 411)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.shape, train_v.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re, string\n\nre_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')\ndef tokenize(s): \n    return re_tok.sub(r' \\1 ', s).split()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### Stop words doesn't increase score","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# from stop_words import get_stop_words\n# stop_words = get_stop_words('spanish') + get_stop_words('turkish')+ get_stop_words('italian')\n# stop_words = set(stop_words)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"COMMENT = 'comment_text'\nLABEL = 'toxic'","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"We can try using a binary version of tf-idf because toxic words influence their presence","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"binary = False","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vec_train = TfidfVectorizer(ngram_range=(1,1), tokenizer=tokenize,\n               min_df=2, max_df=0.9, strip_accents='unicode', use_idf=1, binary=binary,\n               smooth_idf=1, sublinear_tf=1 )\nvec_train.fit(train[COMMENT])\nval_on_train = vec_train.transform(val[COMMENT])\ntrn_on_train = vec_train.transform(train[COMMENT])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"vec_val = TfidfVectorizer(ngram_range=(1,1), tokenizer=tokenize,\n               min_df=2, max_df=0.9, strip_accents='unicode', use_idf=1, binary=binary,\n               smooth_idf=1, sublinear_tf=1 )\n\nvec_val.fit(pd.concat([train_v[COMMENT], val[COMMENT]]))\ntrn_on_val = vec_val.transform(train_v[COMMENT])\nval_on_val = vec_val.transform(val[COMMENT])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_on_train, trn_on_train, trn_on_val, val_on_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_on_train = trn_on_train\nval_x_on_train = val_on_train\n\nx_on_val = trn_on_val\nval_x_on_val = val_on_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_val = val[LABEL].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pr_train(y_i, y):\n    p = x_on_train[y==y_i].sum(0)\n    return (p+3) / ((y==y_i).sum()+3)\n\ndef pr_val(y_i, y):\n    p = x_on_val[y==y_i].sum(0)\n    return (p+0.2) / ((y==y_i).sum()+0.2)\n\ny_train = train[LABEL].values\ny_train_v = train_v[LABEL].values\n\nr_train = np.log(pr_train(1,y_train) / pr_train(0,y_train))\nr_val = np.log(pr_val(1,y_train_v) / pr_val(0,y_train_v))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_nb_on_train = x_on_train.multiply(r_train)\n\nC_PARAMETERS = [1.5, 2, 4]\nmodels_train = [LogisticRegression(C=c, dual=True, solver='liblinear') for c in C_PARAMETERS]\npreds_val_on_train = []\n\nfor model in models_train:\n    model.fit(x_nb_on_train, y_train)\n    p = model.predict_proba(val_x_on_train.multiply(r_train))[:,1]\n    print(roc_auc_score(y_val, p))\n    preds_val_on_train.append(p)\n    \npreds_ensemble_val_on_train = 2**((np.log2(preds_val_on_train[1]) + np.log2(preds_val_on_train[2]))/2)\n\nprint(roc_auc_score(y_val, preds_ensemble_val_on_train))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_nb_on_val = x_on_val.multiply(r_val)\n\nmodels_train_v = [LogisticRegression(C=c, dual=True, solver='liblinear') for c in C_PARAMETERS]\npreds_val_on_val = []\n\nfor model in models_train_v:\n    model.fit(x_nb_on_val, y_train_v)\n    p = model.predict_proba(val_x_on_val.multiply(r_val))[:,1]\n    print(roc_auc_score(y_val, p))\n    preds_val_on_val.append(p)\n    \npreds_ensemble_val_on_val = 2**((np.log2(preds_val_on_val[1]) + np.log2(preds_val_on_val[2]))/2)\n\nprint(roc_auc_score(y_val, preds_ensemble_val_on_val))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_val = [\n    models_train[1].predict_proba(val_x_on_train.multiply(r_train))[:,1],\n    models_train[2].predict_proba(val_x_on_train.multiply(r_train))[:,1],\n    models_train_v[1].predict_proba(val_x_on_val.multiply(r_val))[:,1],\n    models_train_v[2].predict_proba(val_x_on_val.multiply(r_val))[:,1],\n]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_val_ens = 2**np.mean([np.log2(p) for p in preds_val], axis = 0)\nprint(roc_auc_score(y_val, preds_val_ens))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"is_val_lang = test.lang.isin(['tr', 'es', 'it'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"np.sum(is_val_lang.values)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_val_lang = test.loc[is_val_lang, COMMENT]\ntest_non_val_lang = test.loc[~is_val_lang, COMMENT]\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_val_lang_on_train = vec_train.transform(test_val_lang)\ntest_nonval_lang_on_train = vec_train.transform(test_non_val_lang)\ntest_val_lang_on_val = vec_val.transform(test_val_lang)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_val_lang_on_train, test_nonval_lang_on_train, test_val_lang_on_val","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1_val_lang_on_train = models_train[0].predict_proba(test_val_lang_on_train.multiply(r_train))[:,1]\npreds2_val_lang_on_train = models_train[1].predict_proba(test_val_lang_on_train.multiply(r_train))[:,1]\npreds4_val_lang_on_train = models_train[2].predict_proba(test_val_lang_on_train.multiply(r_train))[:,1]\n\npreds1_nonval_lang_on_train = models_train[0].predict_proba(test_nonval_lang_on_train.multiply(r_train))[:,1]\npreds2_nonval_lang_on_train = models_train[1].predict_proba(test_nonval_lang_on_train.multiply(r_train))[:,1]\npreds4_nonval_lang_on_train = models_train[2].predict_proba(test_nonval_lang_on_train.multiply(r_train))[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds1_val_lang_on_val = models_train_v[0].predict_proba(test_val_lang_on_val.multiply(r_val))[:,1]\npreds2_val_lang_on_val = models_train_v[1].predict_proba(test_val_lang_on_val.multiply(r_val))[:,1]\npreds4_val_lang_on_val = models_train_v[2].predict_proba(test_val_lang_on_val.multiply(r_val))[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\npreds_val = 2**((np.log2(preds2_val_lang_on_train) + np.log2(preds4_val_lang_on_train) +np.log2(preds2_val_lang_on_val) + np.log2(preds4_val_lang_on_val)) / 4)\npreds_nonval = 2**((np.log2(preds2_nonval_lang_on_train) + np.log2(preds4_nonval_lang_on_train)) / 2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.loc[is_val_lang, 'toxic'] = preds_val\ntest.loc[~is_val_lang, 'toxic'] = preds_nonval","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.iloc[28].toxic, test.iloc[28].content","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1 = test[['id', 'toxic', 'lang']]\nsubmission1.to_csv('submission1.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Ensembling with a non-ensamble kernel:\n    https://www.kaggle.com/shonenkov/tpu-inference-super-fast-xlmroberta","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission2 = pd.read_csv('../input/tpu-inference-super-fast-xlmroberta/submission.csv') # Ver 4","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1['toxic'] = submission1['toxic'] * 0.04 + submission2['toxic'] * 0.96","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I do not like this method, but it works (+0.0008 9461 -> 9469). \nIt would be better to use mean language-dependent AUC, not AUC for 6 lang. In that case language weights probing wouldn't work\n\nhttps://www.kaggle.com/c/jigsaw-multilingual-toxic-comment-classification/discussion/160980","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1.loc[submission1[\"lang\"] == \"es\", \"toxic\"] *= 1.06\nsubmission1.loc[submission1[\"lang\"] == \"fr\", \"toxic\"] *= 1.04\nsubmission1.loc[submission1[\"lang\"] == \"it\", \"toxic\"] *= 0.97\nsubmission1.loc[submission1[\"lang\"] == \"pt\", \"toxic\"] *= 0.96\nsubmission1.loc[submission1[\"lang\"] == \"tr\", \"toxic\"] *= 0.98","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission1[['id', 'toxic']].to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}