{"cells":[{"metadata":{},"cell_type":"markdown","source":"The task of this notebook is to analyse this kaggle task performance on different models.\n\nModels used:\n\n1)logistic regression\n\n2)SVM\n\n3)Decision tress\n\n#you can similarly try it on Naive bayes classifier and Random forest classifier same as in 1,2,3.\n\n4)Bidirectional LSTM model\n\n5)Bert","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf \nimport pandas as pd\nimport numpy as np","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"try:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# EDA and simple classification model analysis by loss functiion and ROC_AUC_Score\n\nLets use only single dataset from listed ones for this analysis to save time and computation.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"df=pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\ndf.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=df[:1000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df.info() #checking missing data and datype in columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"df=df.drop(columns=['severe_toxic','obscene','threat','insult','identity_hate'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y=df['toxic'].values   #target data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#cleaning the text in train data\nimport re\n\ndef clean_text(data):\n   x=re.sub(r'https?://\\S+|www\\.\\S+','',data) #hyperlinks\n   x=re.sub(r'[!@#$\"]','',x) #symbols\n   x=re.sub(r'\\d+','', x) #digits\n   return x\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"X=[clean_text(i) for i in df['comment_text']]\nX=np.array(X)   \nX[5]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import nltk\nnltk.download('stopwords')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from nltk.corpus import stopwords\n\nt_x=[]\nSTOPWORDS=stopwords.words('english')\nz=[]\nfor i in X:\n  for word in i.split():\n    if word not in STOPWORDS:\n      z+=word\n      z+=' '\n  t_x.append(''.join(z))\n  z=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_x=np.array(t_x) #required feature data","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_x[0:3]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"feature_x.shape,y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.model_selection import train_test_split\n\nx_train,x_test,y_train,y_test=train_test_split(feature_x,y,test_size=0.3,random_state=32)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.feature_extraction.text import TfidfVectorizer,CountVectorizer\n\ntfidf=TfidfVectorizer(strip_accents='unicode', analyzer='word',token_pattern=r'\\w{1,}',\n            ngram_range=(1, 3), use_idf=1,smooth_idf=1,sublinear_tf=1,\n            stop_words = 'english')\n\ntable_tf=tfidf.fit(list(x_train)+list(x_test))\ntable_tf1=tfidf.transform(x_train)\ntable_tf2=tfidf.transform(x_test)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# it is not used but its performance can also be checked by replacing tfidfvectorizer\n\"\"\"vec=CountVectorizer(analyzer='word',token_pattern=r'\\w{1,}', #4 gram \n            ngram_range=(1, 4), stop_words = 'english')\n\ntable_c=vec.fit_transform(list(x_train)+list(x_test))\ntable_c1=vec.transform(x_train)\ntable_c2=vec.transform(x_test)\"\"\"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"table_tf1.shape,y_train.shape","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Implementation of simple classification models for toxic classification","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.linear_model import LogisticRegression \n\nclassifier=LogisticRegression()\nclassifier.fit(table_tf1,y_train)\npreds=classifier.predict_proba(table_tf2)\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"preds[:1]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import log_loss  #loss value\nloss=log_loss(y_test, preds)\nprint(loss)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score    #roc_auc_score\nscore=roc_auc_score(y_test,preds.argmax(axis=1))\nscore","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#applying SVM\n\nfrom sklearn.svm import SVC\nclassifier_1=SVC(probability=True)\nclassifier_1.fit(table_tf1,y_train)\npreds_2=classifier_1.predict_proba(table_tf2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_2=log_loss(y_test,preds_2)  #loss value\nprint(loss_2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score   #roc_auc_score\nscore=roc_auc_score(y_test,preds_2.argmax(axis=1))\nscore","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#applying decision tree\n\nfrom sklearn.tree import DecisionTreeClassifier\n\nclassifier_2=DecisionTreeClassifier()\nclassifier_2.fit(table_tf1,y_train)\npreds_3=classifier_2.predict_proba(table_tf2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"loss_3=log_loss(y_test,preds_2) #loss value\nprint(loss_3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from sklearn.metrics import roc_auc_score   #roc_auc_score\nscore=roc_auc_score(y_test,preds_3.argmax(axis=1))\nscore","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Since we want to to check the performance on other models, we now shift to use neural architecture model with intention of improving the performance over classifying toxic comments.","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#keras model using pre-trained glove embeddings","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n\nfrom tensorflow.keras.layers import Input,Dense,LSTM,SpatialDropout1D,Bidirectional,Dropout,TimeDistributed,Embedding\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.preprocessing.text import Tokenizer\nfrom tensorflow.keras.preprocessing.sequence import pad_sequences\nfrom tensorflow.keras.regularizers import Regularizer\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer=Tokenizer()\ntokenizer.fit_on_texts(x_train)\nsequence=tokenizer.texts_to_sequences(x_train)\n\nword_index=tokenizer.word_index\nvocab_len=len(tokenizer.word_index)\nmax_len=max([len(i) for i in sequence])\n\nsequences=pad_sequences(sequence,maxlen=max_len)      #train_sequences\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#preprocessing of test_sequences\ntest=[]                  \nalpha=[]\n\nfor x in x_test: \n   for i in x.split():\n     if i in word_index.keys():\n       test.append(i)\n   alpha.append(' '.join(test))  \n   test=[]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set=[]\ntest_set_y=[]\nfor i in range(len(alpha)):\n   if alpha[i]!='':\n     test_set.append(alpha[i])\n     test_set_y.append(y_test[i])\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set[:2],test_set_y[:2]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set_y=np.array(test_set_y)\ntest_set_y.shape  #target test_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_set=tokenizer.texts_to_sequences(test_set)\n\nsequence_test=pad_sequences(test_set,maxlen=max_len) #test_sequences","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sequence_test.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#using transfer learning\n#loading pre-trained glove model\n\n!wget https://storage.googleapis.com/laurencemoroney-blog.appspot.com/glove.6B.100d.txt","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_dim=100\nembedding_index = {};\nwith open('./glove.6B.100d.txt') as f:\n    for line in f:\n        values = line.split();\n        word = values[0];\n        coefs = np.asarray(values[1:], dtype='float32');\n        embedding_index[word] = coefs;\n\nembedding_mat = np.zeros((vocab_len+1, embedding_dim));\nfor word, i in word_index.items():\n    if word in list(embedding_index.keys()):\n      if i!=7581:\n        embedding_mat[i]=embedding_index.get(word)\n  \n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"sequences.shape,sequence_test.shape,y_train.shape,test_set_y.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam\n\n#building model\n\ni=Input(shape=(819,))\nx=Embedding(vocab_len+1,embedding_dim,weights=[embedding_mat],trainable=False)(i)\nx=Bidirectional(LSTM(512))(x)\nx=Dropout(0.2)(x)\nx=Dense(256,activation='relu')(x)\nx=Dropout(0.2)(x)\nx=Dense(1,activation='sigmoid')(x)\n\nmodel=Model(i,x)\n\nmodel.compile(optimizer=Adam(0.001),loss='binary_crossentropy',metrics=['binary_accuracy'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"r=model.fit(sequences,y_train,validation_data=(sequence_test,test_set_y),epochs=3,verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"I computed upto only 3 epochs because as it was taking a lot of time for computations.\n\nWe need more epochs for conclusion of the model quality and accordingly we can determine that our model has a good fit,overfitting or underfitting and then we can tune our hyperparameters in our model to get the best results","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nplt.plot(r.history['loss'],label='loss')\nplt.plot(r.history['val_loss'],label='val_loss')\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Now lets apply the bert model as asked in the competition to see the losses","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import transformers\nfrom transformers import AutoModel, AutoTokenizer, BertTokenizer\nfrom tokenizers import BertWordPieceTokenizer\nfrom transformers import AutoModel, AutoTokenizer, BertTokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import pandas as pd\ntrain_x1=pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\ntrain_x2=pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv')\ntest_d=pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nval_d=pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/validation.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x1=train_x1[['comment_text','toxic']]\ntrain_x1.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_x2=train_x2[['comment_text','toxic']]\ntrain_x2.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(train_x2[train_x2['toxic']==1]) #checking","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_train_d=pd.concat([train_x1,train_x2[train_x2['toxic']==1],train_x2[train_x2['toxic']==0]])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"final_train_d=final_train_d[:1000]\nfinal_train_d.head(1)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_d=test_d[['content','lang']]\ntest_d=test_d[:1000]\ntest_d.head(3)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_d=val_d[['comment_text','lang','toxic']]\nval_d=val_d[:1000]\nval_d.head(3)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Help taken from :\n\n1)http://jalammar.github.io/illustrated-bert/\n\n2)https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n\n3)https://huggingface.co/transformers/","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"import tensorflow as tf\n#IMP DATA FOR CONFIG\n\nAUTO = tf.data.experimental.AUTOTUNE\n\n\n# Configuration\nEPOCHS = 3\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nMAX_LEN = 192","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\"\"\"\n\ndef fast_encode(texts, tokenizer, chunk_size=256, maxlen=512):\n\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n   \n    all_ids = []\n    \n    for i in range(0, len(texts), chunk_size):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tokenizer=transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\ntokenizer.save_pretrained('.')\n\nfast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\nfast_tokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"x_train_d=fast_encode(final_train_d['comment_text'].astype(str),fast_tokenizer,maxlen=MAX_LEN)\nx_valid_d=fast_encode(val_d['comment_text'].astype(str),fast_tokenizer,maxlen=MAX_LEN)\nx_test_d=fast_encode(test_d['content'].astype(str),fast_tokenizer,maxlen=MAX_LEN)\ntype(x_train_d)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"y_train_d=final_train_d['toxic'].values\ny_valid_d=val_d['toxic'].values\ny_test_d=test_d['lang'].values","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset=(tf.data.Dataset\n              .from_tensor_slices((x_train_d,y_train_d))\n              .repeat()\n              .shuffle(2048)\n              .batch(BATCH_SIZE)\n              .cache()\n              .prefetch(AUTO))\n\nvalid_dataset = (tf.data.Dataset\n    .from_tensor_slices((x_valid_d, y_valid_d))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test_d)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tensorflow.keras.optimizers import Adam\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#building_model\n\ndef build_model(transformer, max_len=512):\n    \"\"\"\n    https://www.kaggle.com/xhlulu/jigsaw-tpu-distilbert-with-huggingface-and-keras\n    \"\"\"\n    input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n    sequence_output = transformer(input_word_ids)[0]\n    cls_token = sequence_output[:, 0, :]\n    out = Dense(1, activation='sigmoid')(cls_token)\n    \n    model = Model(inputs=input_word_ids, outputs=out)\n    model.compile(Adam(lr=1e-5), loss='binary_crossentropy', metrics=['accuracy'])\n    \n    return model","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"with strategy.scope():\n    transformer_layer = (\n        transformers.TFDistilBertModel\n        .from_pretrained('distilbert-base-multilingual-cased')\n    )\n    model = build_model(transformer_layer, max_len=MAX_LEN)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = x_train_d.shape[0] // 100\nr=model.fit(train_dataset,steps_per_epoch=n_steps,validation_data=valid_dataset,epochs=5)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"plt.plot(r.history['loss'],label='loss')\nplt.plot(r.history['accuracy'],label='accuracy')\n\nplt.legend()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}