{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install pymorphy2","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np \nimport pandas as pd \nimport pymorphy2\nimport gensim\nimport itertools\nimport tensorflow as tf\nimport xgboost as xgb\nfrom collections import Counter,defaultdict\nfrom pymorphy2 import MorphAnalyzer\nfrom gensim.utils import tokenize\nfrom nltk.corpus import stopwords\nfrom sklearn.linear_model import LogisticRegression\nfrom sklearn.model_selection import train_test_split, cross_val_score\nfrom sklearn.metrics import classification_report, confusion_matrix\nfrom sklearn.metrics import roc_auc_score\nfrom string import punctuation, digits\nimport seaborn as sns\nfrom mlxtend.plotting import plot_learning_curves\nfrom mlxtend.plotting import plot_decision_regions\nimport matplotlib.pyplot as plt\nimport matplotlib.gridspec as gridspec\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.naive_bayes import GaussianNB \nfrom sklearn.ensemble import RandomForestClassifier\nfrom mlxtend.classifier import StackingClassifier\n\n\nfrom sklearn.tree import DecisionTreeClassifier\nfrom sklearn.neighbors import KNeighborsClassifier\nfrom sklearn.linear_model import LogisticRegression\n\nfrom sklearn.ensemble import AdaBoostClassifier\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"data = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Validation","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"valid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid.lang.unique()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Перевожу test и valid на английский","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Test","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"!pip install googletrans","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#from googletrans import Translator\n#translator = Translator()\n\n#translated = []\n#for idx in range(len(test)):\n#     translated.append(translator.translate(test.content[idx], src=test.lang[idx]))\n        \n#test['translated'] = translated","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#test.to_csv('test_translated.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_translated = pd.read_csv('../input/translated/test_translated.csv')[['content', 'lang', 'translated']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_translated.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Valid я не стала считать, взяла у Лизы Носовой посчитанное #ml_project_sharing","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_translated = pd.read_csv('../input/translated/validation_with_translation.csv')[['comment_text', 'lang', 'toxic', 'translation']]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid_translated.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Делаю эмбеддинги и предсказываю `toxic`","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### 1. Пробую сделать вектора в word2vec и fasttext и посчитать roc_auc логистической регрессии на трейне и валидационной выборке\n\n(это пункт из 2 строки \"Бейзлайн модель, которая работает на отложенной выборке (train_test_split) и validation.csv (различие в точности может быть большим, но главное попробовать любыми способами его уменьшить и зафиксировать эксперименты) - 3 балла\")","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"morph = MorphAnalyzer()\nstops = set(stopwords.words('english'))\n\ndef normalize(sent):\n    tokens = list(tokenize(sent))\n    norm_tokens = [morph.parse(word)[0].normal_form for word in tokens if word and word not in stops]\n    return norm_tokens","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data['comment_norm'] = [normalize(x) for x in data.comment_text]\ntest_translated['translated_norm'] = [normalize(x) for x in test_translated.translated]\nvalid_translated['translated_norm'] = [normalize(x) for x in valid_translated.translation]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fast_text = gensim.models.FastText(data.comment_norm, size=50, min_n=4, max_n=8) \nw2v = gensim.models.Word2Vec(data.comment_norm, size=50, sg=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def get_embedding(text, model, dim):\n    words = Counter(text)\n    total = len(text)\n    vectors = np.zeros((len(words), dim))\n    \n    for i,word in enumerate(words):\n        try:\n            v = model[word]\n            vectors[i] = v*(words[word]/total)\n        except (KeyError, ValueError):\n            continue\n    \n    if vectors.any():\n        vector = np.average(vectors, axis=0)\n    else:\n        vector = np.zeros((dim))\n    \n    return vector","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def vectorize(data, model, dim=50):\n    X = np.zeros((len(data), dim))\n    for i, text in enumerate(data.values):\n        X[i] = get_embedding(text, model, dim)\n\n    return X","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X, valid_X, train_y, valid_y = train_test_split(vectorize(data.comment_norm, w2v), data.toxic, random_state=1)\nclf = LogisticRegression(C=1000, max_iter=500, class_weight='balanced')\nclf.fit(train_X, train_y)\npreds = clf.predict_proba(valid_X)\npred = [x[1] for x in preds]\nprint(roc_auc_score(valid_y, pred))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_X, valid_X, train_y, valid_y = train_test_split(vectorize(data.comment_norm, fast_text), data.toxic, random_state=1)\nclf = LogisticRegression(C=1000,max_iter=500, class_weight='balanced')\nclf.fit(train_X, train_y)\npreds = clf.predict_proba(valid_X)\npred = [x[1] for x in preds]\nprint(roc_auc_score(valid_y, pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_vec = vectorize(valid_translated.translated_norm, w2v)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_valid = clf.predict_proba(val_vec)\npred = [x[1] for x in preds_valid]\nprint(roc_auc_score(list(valid_translated.toxic), pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_vec2 = vectorize(valid_translated.translated_norm, fast_text)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_valid = clf.predict_proba(val_vec2)\npred = [x[1] for x in preds_valid]\nprint(roc_auc_score(list(valid_translated.toxic), pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Word2vec показывает лучшее качество как на обучающей, так и на валидационной выборке.\n\nэто пункт \"Сравнение нескольких видов эмбедингов по отдельности - 1.5 балла\" из строки 5","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"#### 2. Пробую сделать стакинг бустингов\n\nСтакинг бустингов (как минимум 2 уровня xboost/catboost/lightgbm, можно использовать публичные кернелы любых соревнований) - 3 балла","execution_count":null},{"metadata":{},"cell_type":"markdown","source":"Adaboost","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"clf = DecisionTreeClassifier(criterion='entropy', max_depth=3)\n\nnum_est = [3, 5, 7, 10]\nlabels = ['AdaBoost (n_est=3)', 'AdaBoost (n_est=5)', 'AdaBoost (n_est=7)', 'AdaBoost (n_est=10)']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"for n_est, label in zip(num_est, labels):   \n    boosting = AdaBoostClassifier(base_estimator=clf, n_estimators=n_est)   \n    boosting.fit(train_X, train_y)\n    preds_valid = boosting.predict_proba(valid_X)\n    pred = [x[1] for x in preds_valid]\n    print(label, roc_auc_score(valid_y, pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_valid = boosting.predict_proba(val_vec2)\npred = [x[1] for x in preds_valid]\nprint(roc_auc_score(list(valid_translated.toxic), pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"XGboost","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"clf = xgb.XGBClassifier(objective='binary:logistic')\nclf.fit(train_X,  train_y)\npreds_valid = clf.predict_proba(valid_X)\npred = [x[1] for x in preds_valid]\nprint(roc_auc_score(valid_y, pred))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds_valid = clf.predict_proba(val_vec2)\npred = [x[1] for x in preds_valid]\nprint(roc_auc_score(list(valid_translated.toxic), pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Stacking","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"clf = DecisionTreeClassifier(criterion='entropy', max_depth=4)\nlr = LogisticRegression(C=1000, max_iter=500, class_weight='balanced')\n\nclf1 = xgb.XGBClassifier(objective='binary:logistic')\nclf2 = AdaBoostClassifier(base_estimator=clf, n_estimators=8)\n\nsclf = StackingClassifier(classifiers=[clf1, clf2], meta_classifier=lr)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"labels = ['XGboost', 'AdaBoost', 'Stacking Classifier']\nclf_list = [clf1, clf2, sclf]\n\nfor clf, label in zip(clf_list, labels):\n        \n    scores = cross_val_score(clf, train_X, train_y, cv=3, scoring='roc_auc')\n    print (\"Roc Auc: %.2f (+/- %.2f) [%s]\" %(scores.mean(), scores.std(), label))\n\n    clf.fit(train_X, train_y)\n    \n    preds_valid = clf.predict_proba(valid_X)\n    pred = [x[1] for x in preds_valid]\n    print(label, roc_auc_score(valid_y, pred))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 3. Пробую сделать собственную нейросеть\n\nЛюбая нейронная модель (минимум 5 слоев) с Dropout, Pooling и колбеками - 2 балла","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def prepareEmbeddings(texts):\n    vocab = Counter()\n    for text in texts:\n        vocab.update(text)\n        \n    filtered_vocab = set()\n    for word in vocab:\n        if vocab[word] > 5:\n            filtered_vocab.add(word)\n    \n    word2id = {'UNK':1, 'PAD':0}\n    for word in filtered_vocab:\n        word2id[word] = len(word2id)\n    \n    id2word = {i:word for word, i in word2id.items()}\n\n    X = []\n    for text in texts:\n        tokens = text\n        ids = [word2id.get(token, 1) for token in tokens]\n        X.append(ids)\n    \n    X = tf.keras.preprocessing.sequence.pad_sequences(X, maxlen=200)\n    return X, word2id","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_train, word2id = prepareEmbeddings(data.comment_norm)\ny_train = np.array(data.toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X_valid, valword2id = prepareEmbeddings(valid_translated.translated_norm)\ny_valid = np.array(valid_translated.toxic)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"checkpoint = tf.keras.callbacks.ModelCheckpoint('model.weights', # названия файла \n                                                monitor='val_auc', # за какой метрикой следить\n                                                verbose=1, # будет печатать что происходит\n                                                save_weights_only=True, # если нужно только веса сохранить\n                                                save_best_only=True, # сохранять только лучшие\n                                                mode='max', # если метрика должна расти, то тут max и min если наоборот\n                                                save_freq='epoch' # как часто вызывать\n                                               )\n\nearly_stop = tf.keras.callbacks.EarlyStopping(monitor='val_auc', \n                                              min_delta=0.01, # какая разница считается как улучшение\n                                              patience=2, # сколько эпох терпеть отсутствие улучшений\n                                              verbose=1, \n                                              mode='max',\n                                              )","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inputs = tf.keras.layers.Input(shape=(200,))\nembeddings = tf.keras.layers.Embedding(input_dim=len(word2id), output_dim=100)(inputs, )\n\ndrop1 = tf.keras.layers.Dropout(0.4)(embeddings)\nconv1 = tf.keras.layers.Conv1D(kernel_size=3, filters=32, strides=1, kernel_regularizer='l2', activation='relu')(drop1)\nconv2 = tf.keras.layers.Conv1D(kernel_size=5, filters=32, strides=2, kernel_regularizer='l2', activation='relu')(conv1)\npool = tf.keras.layers.AveragePooling1D()(conv2)\ndrop2 = tf.keras.layers.Dropout(0.3)(pool)\n\nflatten = tf.keras.layers.Flatten()(drop2)\ndense = tf.keras.layers.Dense(50, activation='relu')(flatten)\noutputs = tf.keras.layers.Dense(1, activation='sigmoid')(dense)\n\nmodel = tf.keras.Model(inputs=inputs, outputs=outputs)\noptimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\nmodel.compile(optimizer=optimizer,\n              loss='binary_crossentropy',\n              metrics='AUC')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X_train, y_train, \n          validation_data=(X_valid, y_valid),\n          batch_size=2000,\n          epochs=5,\n          callbacks=[checkpoint, early_stop])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(model.history.history.keys())\nplt.plot(model.history.history['auc'])\nplt.plot(model.history.history['val_auc'])\nplt.title('model f1')\nplt.ylabel('auc')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"inputs = tf.keras.layers.Input(shape=(200,))\nembeddings = tf.keras.layers.Embedding(input_dim=len(word2id), output_dim=100)(inputs, )\n\n# kernel_size = 3\npad1 = tf.keras.layers.Lambda(lambda x: tf.pad(x, [[0,0], [1,1], [0, 0]], mode='REFLECT'))(embeddings)\n\nconv1 = tf.keras.layers.Conv1D(kernel_size=3, filters=32, strides=1)(pad1)\ndrop1 = tf.keras.layers.Dropout(0.3)(conv1)\n\npad2 = tf.keras.layers.Lambda(lambda x: tf.pad(x, [[0,0], [1,1], [0, 0]], mode='REFLECT'))(drop1)\n\nconv2 = tf.keras.layers.Conv1D(kernel_size=3, filters=32,strides=1, kernel_regularizer='l2', activation='relu')(pad2)\npool1 = tf.keras.layers.AveragePooling1D()(conv2)\nconv3 = tf.keras.layers.Conv1D(kernel_size=3, filters=32,strides=1, kernel_regularizer='l2', activation='relu')(pool1)\n\n#kernel_size = 5\npad3 = tf.keras.layers.Lambda(lambda x: tf.pad(x, [[0,0], [2,2], [0, 0]],mode='REFLECT'))(embeddings)\n\nconv4 = tf.keras.layers.Conv1D(kernel_size=5, filters=32, strides=1)(pad3)\ndrop2 = tf.keras.layers.Dropout(0.3)(conv4)\n\npad4 = tf.keras.layers.Lambda(lambda x: tf.pad(x, [[0,0], [2,2], [0, 0]],mode='REFLECT'))(drop2)\n\nconv5 = tf.keras.layers.Conv1D(kernel_size=5, filters=32,strides=1, kernel_regularizer='l2', activation='relu')(pad4)\npool2 = tf.keras.layers.AveragePooling1D()(conv5)\nconv6 = tf.keras.layers.Conv1D(kernel_size=3, filters=32,strides=1, kernel_regularizer='l2', activation='relu')(pool2)\n\n\nconcat = tf.keras.layers.concatenate([conv3, conv6])\ndrop3 = tf.keras.layers.Dropout(0.5)(concat)\n\nconv_global = tf.keras.layers.Conv1D(kernel_size=5, filters=32, strides=1)(drop3)\nflatten = tf.keras.layers.Flatten()(conv_global)\ndense = tf.keras.layers.Dense(50, activation='relu')(flatten)\ndense2 = tf.keras.layers.Dense(25, activation='relu')(dense)\noutputs = tf.keras.layers.Dense(1, activation='sigmoid')(dense2)\n\nmodel = tf.keras.Model(inputs=inputs, outputs=outputs)\noptimizer = tf.keras.optimizers.Adam(learning_rate=0.001)\nmodel.compile(optimizer=optimizer,\n              loss='binary_crossentropy',\n              metrics='AUC')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X_train, y_train, \n          validation_data=(X_valid, y_valid),\n          batch_size=2000,\n          epochs=10,\n          callbacks=[checkpoint, early_stop])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(model.history.history.keys())\nplt.plot(model.history.history['auc'])\nplt.plot(model.history.history['val_auc'])\nplt.title('model f1')\nplt.ylabel('auc')\nplt.xlabel('epoch')\nplt.legend(['train', 'val'], loc='upper left')\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"#### 4. Делаю сабмит\n\nСамбит на соревнование тетрадки с любой нейронкой и ненулевым результатом (должна быть полностью написана вами - предобработка, модель, валидация. Куски с созданием сабмишена можно брать из других кернелов) - 2.5 балла","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"test_dataset, word2idtest = prepareEmbeddings(test_translated.translated)\ntest_dataset","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"idx = list(test_translated.index)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"s = model.predict(test_dataset, verbose=1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"s = [float(x) for x in s]\ns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub = pd.DataFrame(zip(idx, s), columns=['id', 'toxic'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sub.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Score = 0.4909","execution_count":null}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}