{"cells":[{"metadata":{},"cell_type":"markdown","source":"#### Please refer to the original kernel: https://www.kaggle.com/jhoward/nb-svm-strong-linear-baseline","execution_count":null},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import pandas as pd, numpy as np\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\nfrom scipy.sparse import hstack\nimport gc\nimport os\n#for dirname, _, filenames in os.walk('/kaggle/input'):\n#    for filename in filenames:\n#        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"markdown","source":"train1 = pd.read_csv(\"/kaggle/input/jigsaw-train-translated/train_mic.csv\")\n#train2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n#train2.toxic = train2.toxic.round().astype(int)\n\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv')\n#valid1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"valid = pd.concat([\n    valid[['comment_text', 'toxic']],\n    valid[['translated', 'toxic']].rename(columns={'translated': 'comment_text'})])\n\n#valid['comment_text'] = valid['translated'] #+' '+valid1['comment_text']","execution_count":null},{"metadata":{"trusted":true},"cell_type":"markdown","source":"train = pd.concat([\n    train1[['comment_text', 'toxic']],\n    train1[['tr', 'toxic']].rename(columns={'tr': 'comment_text'}).dropna(),\n    train1[['ru', 'toxic']].rename(columns={'ru': 'comment_text'}).dropna(),\n    train1[['it', 'toxic']].rename(columns={'it': 'comment_text'}).dropna(),\n    train1[['fr', 'toxic']].rename(columns={'fr': 'comment_text'}).dropna(),\n    train1[['pt', 'toxic']].rename(columns={'pt': 'comment_text'}).dropna(),\n    train1[['es', 'toxic']].rename(columns={'es': 'comment_text'}).dropna(),\n    #train2[['comment_text', 'toxic']].query('toxic==1'),\n    #train2[['comment_text', 'toxic']].query('toxic==0'),\n    valid[['comment_text', 'toxic']]\n    #valid1[['comment_text', 'toxic']]\n]).reset_index(drop=True)","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Loading data\n\ntrain1 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\ntrain1['lang'] = 'en'\n\ntrain_es = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-es-cleaned.csv')\ntrain_es['lang'] = 'es'\n\ntrain_fr = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-fr-cleaned.csv')\ntrain_fr['lang'] = 'fr'\n\ntrain_pt = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-pt-cleaned.csv')\ntrain_pt['lang'] = 'pt'\n\ntrain_ru = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-ru-cleaned.csv')\ntrain_ru['lang'] = 'ru'\n\ntrain_it = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-it-cleaned.csv')\ntrain_it['lang'] = 'it'\n\ntrain_tr = pd.read_csv('/kaggle/input/jigsaw-train-multilingual-coments-google-api/jigsaw-toxic-comment-train-google-tr-cleaned.csv')\ntrain_tr['lang'] = 'tr'\n\n#train2 = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv\")\n#train2.toxic = train2.toxic.round().astype(int)\n#train2['lang'] = 'en'\n\ntrain = pd.concat([\n    \n    train1[['comment_text', 'lang', 'toxic']],\n    train_es[['comment_text', 'lang', 'toxic']],\n    train_tr[['comment_text', 'lang', 'toxic']],\n    train_fr[['comment_text', 'lang', 'toxic']],\n    train_pt[['comment_text', 'lang', 'toxic']],\n    train_ru[['comment_text', 'lang', 'toxic']],\n    train_it[['comment_text', 'lang', 'toxic']]\n    \n]).sample(n=300000).reset_index(drop=True)\n\ndel train1, train_es, train_fr, train_pt, train_ru, train_it, train_tr\ngc.collect()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\n#train1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv')\n\n#valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_valid_translated.csv')\n#valid1 = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\n\n#test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-test-translated/jigsaw_miltilingual_test_translated.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\n\nsubm = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#train = pd.concat([train,train1])\n#train = pd.concat([train,valid])\n#valid['comment_text'] = valid['translated'] #+' '+valid1['comment_text']\n#test['content'] = test['translated'] #+' '+test1['content']\n#train = pd.concat([train,valid])\n#train = valid.copy()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Looking at the data\n## The training data contains a row per comment, with an id, the text of the comment, and 6 different labels that we'll try to predict.","execution_count":null},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['comment_text'][0]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['comment_text'][2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lens = train.comment_text.str.len()\nlens.mean(), lens.std(), lens.max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lens.hist();","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"label_cols = ['toxic']\ntrain['none'] = 1-train[label_cols].max(axis=1)\ntrain.describe()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train),len(test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train['comment_text'].fillna(\"unknown\", inplace=True)\ntest['content'].fillna(\"unknown\", inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re, string\nre_tok = re.compile(f'([{string.punctuation}“”¨«»®´·º½¾¿¡§£₤‘’])')\ndef tokenize(s): return re_tok.sub(r' \\1 ', s).split()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n = train.shape[0]\n\nvec = TfidfVectorizer(ngram_range=(1,2), tokenizer=tokenize,\n               strip_accents='unicode', use_idf=1,\n               smooth_idf=1, sublinear_tf=1 )\n\n\ntrn_term_doc = vec.fit_transform(train['comment_text'])\ntest_term_doc = vec.transform(test['content'])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"trn_term_doc, test_term_doc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def pr(y_i, y):\n    p = x[y==y_i].sum(0)\n    return (p+1) / ((y==y_i).sum()+1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x = trn_term_doc\ntest_x = test_term_doc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_mdl(y):\n    y = y.values\n    r = np.log(pr(1,y) / pr(0,y))\n    m = LogisticRegression(C=4, dual=False)\n    x_nb = x.multiply(r)\n    return m.fit(x_nb, y), r","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds = np.zeros((len(test), len(label_cols)))\n\nfor i, j in enumerate(label_cols):\n    print('fit', j)\n    m,r = get_mdl(train[j])\n    preds[:,i] = m.predict_proba(test_x.multiply(r))[:,1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submid = pd.DataFrame({'id': subm[\"id\"]})\nsubmission = pd.concat([submid, pd.DataFrame(preds, columns = label_cols)], axis=1)\n\nsubmission.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"submission.head(n=20)","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}