{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"code","source":"\"\"\"\"\nThis is a copy of the notebook Deep Learning for NLP by MR_KNOWNOTHING.\nThe sections with my personal learning notes are marked with:\n\n'>>> My Learning\n\n'<<< End of My Learning\n\"\"\"\"\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU,SimpleRNN\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils import np_utils\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\nfrom keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D\nfrom keras.preprocessing import sequence, text\nfrom keras.callbacks import EarlyStopping\n\n\nimport matplotlib.pyplot as plt\nimport seaborn as sns\n%matplotlib inline\nfrom plotly import graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:55:19.300057Z","iopub.execute_input":"2023-07-28T01:55:19.300441Z","iopub.status.idle":"2023-07-28T01:55:29.157126Z","shell.execute_reply.started":"2023-07-28T01:55:19.300413Z","shell.execute_reply":"2023-07-28T01:55:29.153933Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Configuring TPU's\n\nFor this version of Notebook we will be using TPU's as we have to built a BERT Model","metadata":{}},{"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:55:54.935397Z","iopub.execute_input":"2023-07-28T01:55:54.935819Z","iopub.status.idle":"2023-07-28T01:55:54.956141Z","shell.execute_reply.started":"2023-07-28T01:55:54.935754Z","shell.execute_reply":"2023-07-28T01:55:54.955238Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\nvalidation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:56:07.545074Z","iopub.execute_input":"2023-07-28T01:56:07.545487Z","iopub.status.idle":"2023-07-28T01:56:09.95598Z","shell.execute_reply.started":"2023-07-28T01:56:07.545455Z","shell.execute_reply":"2023-07-28T01:56:09.954916Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Exploring the dataset","metadata":{}},{"cell_type":"markdown","source":"Competition's objective: Build multilingual model from Eng-only training data to predict the probability that a comment is toxic.","metadata":{}},{"cell_type":"markdown","source":"Data from first and second competitions","metadata":{}},{"cell_type":"code","source":"primera = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\nsegunda = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:56:09.958054Z","iopub.execute_input":"2023-07-28T01:56:09.958758Z","iopub.status.idle":"2023-07-28T01:56:42.17646Z","shell.execute_reply.started":"2023-07-28T01:56:09.958718Z","shell.execute_reply":"2023-07-28T01:56:42.175328Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"primera.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:56:42.178089Z","iopub.execute_input":"2023-07-28T01:56:42.178537Z","iopub.status.idle":"2023-07-28T01:56:42.205229Z","shell.execute_reply.started":"2023-07-28T01:56:42.178501Z","shell.execute_reply":"2023-07-28T01:56:42.2042Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"segunda.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:56:42.207391Z","iopub.execute_input":"2023-07-28T01:56:42.207717Z","iopub.status.idle":"2023-07-28T01:56:42.239746Z","shell.execute_reply.started":"2023-07-28T01:56:42.207691Z","shell.execute_reply":"2023-07-28T01:56:42.238575Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training data preprocessed for BERT","metadata":{}},{"cell_type":"code","source":"primera_pre = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv')\nsegunda_pre = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train-processed-seqlen128.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:56:42.241271Z","iopub.execute_input":"2023-07-28T01:56:42.241709Z","iopub.status.idle":"2023-07-28T01:58:19.209086Z","shell.execute_reply.started":"2023-07-28T01:56:42.241669Z","shell.execute_reply":"2023-07-28T01:58:19.208056Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"primera_pre.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.210274Z","iopub.execute_input":"2023-07-28T01:58:19.210586Z","iopub.status.idle":"2023-07-28T01:58:19.226434Z","shell.execute_reply.started":"2023-07-28T01:58:19.21056Z","shell.execute_reply":"2023-07-28T01:58:19.225223Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"pd.options.display.max_colwidth = 200","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.228379Z","iopub.execute_input":"2023-07-28T01:58:19.228876Z","iopub.status.idle":"2023-07-28T01:58:19.241388Z","shell.execute_reply.started":"2023-07-28T01:58:19.228837Z","shell.execute_reply":"2023-07-28T01:58:19.239807Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"primera_pre.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.24299Z","iopub.execute_input":"2023-07-28T01:58:19.243449Z","iopub.status.idle":"2023-07-28T01:58:19.266963Z","shell.execute_reply.started":"2023-07-28T01:58:19.243389Z","shell.execute_reply":"2023-07-28T01:58:19.265856Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Training and testing data for current competition","metadata":{}},{"cell_type":"code","source":"pd.options.display.max_colwidth = 50","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.268649Z","iopub.execute_input":"2023-07-28T01:58:19.269091Z","iopub.status.idle":"2023-07-28T01:58:19.280523Z","shell.execute_reply.started":"2023-07-28T01:58:19.269053Z","shell.execute_reply":"2023-07-28T01:58:19.279008Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train[(train['identity_hate']==1)]","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.285094Z","iopub.execute_input":"2023-07-28T01:58:19.285453Z","iopub.status.idle":"2023-07-28T01:58:19.30703Z","shell.execute_reply.started":"2023-07-28T01:58:19.285425Z","shell.execute_reply":"2023-07-28T01:58:19.30585Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"test.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.308445Z","iopub.execute_input":"2023-07-28T01:58:19.309553Z","iopub.status.idle":"2023-07-28T01:58:19.320575Z","shell.execute_reply.started":"2023-07-28T01:58:19.309311Z","shell.execute_reply":"2023-07-28T01:58:19.319402Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation.head()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.321916Z","iopub.execute_input":"2023-07-28T01:58:19.322306Z","iopub.status.idle":"2023-07-28T01:58:19.339566Z","shell.execute_reply.started":"2023-07-28T01:58:19.32227Z","shell.execute_reply":"2023-07-28T01:58:19.33868Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"validation['comment_text'][(validation['lang']=='es')&(validation['toxic']==1)]","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.341051Z","iopub.execute_input":"2023-07-28T01:58:19.342023Z","iopub.status.idle":"2023-07-28T01:58:19.35196Z","shell.execute_reply.started":"2023-07-28T01:58:19.341969Z","shell.execute_reply":"2023-07-28T01:58:19.351046Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ejemplo = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.353235Z","iopub.execute_input":"2023-07-28T01:58:19.354028Z","iopub.status.idle":"2023-07-28T01:58:19.384739Z","shell.execute_reply.started":"2023-07-28T01:58:19.353995Z","shell.execute_reply":"2023-07-28T01:58:19.383595Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"ejemplo.tail()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.387569Z","iopub.execute_input":"2023-07-28T01:58:19.387989Z","iopub.status.idle":"2023-07-28T01:58:19.398125Z","shell.execute_reply.started":"2023-07-28T01:58:19.38795Z","shell.execute_reply":"2023-07-28T01:58:19.396934Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will drop the other columns and approach this problem as a Binary Classification Problem and also we will have our exercise done on a smaller subsection of the dataset(only 12000 data points) to make it easier to train the models\n","metadata":{}},{"cell_type":"code","source":"train.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.399342Z","iopub.execute_input":"2023-07-28T01:58:19.39963Z","iopub.status.idle":"2023-07-28T01:58:19.42844Z","shell.execute_reply.started":"2023-07-28T01:58:19.399604Z","shell.execute_reply":"2023-07-28T01:58:19.427565Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train = train.loc[:12000,:] # .loc[rows,columns]\ntrain.shape","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.429573Z","iopub.execute_input":"2023-07-28T01:58:19.430584Z","iopub.status.idle":"2023-07-28T01:58:19.436993Z","shell.execute_reply.started":"2023-07-28T01:58:19.430551Z","shell.execute_reply":"2023-07-28T01:58:19.435977Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"code","source":"train.tail()","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.438543Z","iopub.execute_input":"2023-07-28T01:58:19.438934Z","iopub.status.idle":"2023-07-28T01:58:19.455725Z","shell.execute_reply.started":"2023-07-28T01:58:19.438906Z","shell.execute_reply":"2023-07-28T01:58:19.454467Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"We will check the maximum number of words that can be present in a comment , this will help us in padding later","metadata":{}},{"cell_type":"code","source":"train['comment_text'].apply(lambda x:len(str(x).split())).max()\n\n# If separator is not specified for .split, whitespace acts as separator.\n# https://docs.python.org/3/library/stdtypes.html#str.split","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.45717Z","iopub.execute_input":"2023-07-28T01:58:19.457967Z","iopub.status.idle":"2023-07-28T01:58:19.535243Z","shell.execute_reply.started":"2023-07-28T01:58:19.457927Z","shell.execute_reply":"2023-07-28T01:58:19.534124Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"Writing a function for getting auc score for validation","metadata":{}},{"cell_type":"code","source":"def roc_auc(predictions,target):\n    '''\n    This methods returns the AUC Score when given the Predictions\n    and Labels\n    '''\n    \n    fpr, tpr, thresholds = metrics.roc_curve(target, predictions)\n    roc_auc = metrics.auc(fpr, tpr)\n    return roc_auc\n\n# The AUC-ROC curve (Area under the curve - Receiver Operating Characteristic) \n# plots the True Positive rate against the False Positive rate at various threshold settings.\n# It tells you how good the model is to predict 0 as 0 and 1 as 1; the higher the AUC the better.\n\n# https://towardsdatascience.com/understanding-auc-roc-curve-68b2303cc9c5","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.537019Z","iopub.execute_input":"2023-07-28T01:58:19.537317Z","iopub.status.idle":"2023-07-28T01:58:19.542841Z","shell.execute_reply.started":"2023-07-28T01:58:19.537292Z","shell.execute_reply":"2023-07-28T01:58:19.541647Z"},"trusted":true},"execution_count":null,"outputs":[]},{"cell_type":"markdown","source":"# Data preparation","metadata":{}},{"cell_type":"code","source":"xtrain, xvalid, ytrain, yvalid = train_test_split(train.comment_text.values, train.toxic.values, \n                                                  stratify=train.toxic.values, \n                                                  random_state=42, \n                                                  test_size=0.2, shuffle=True)","metadata":{"execution":{"iopub.status.busy":"2023-07-28T01:58:19.544441Z","iopub.execute_input":"2023-07-28T01:58:19.544896Z","iopub.status.idle":"2023-07-28T01:58:19.564367Z","shell.execute_reply.started":"2023-07-28T01:58:19.544854Z","shell.execute_reply":"2023-07-28T01:58:19.563469Z"},"trusted":true},"execution_count":null,"outputs":[]}]}