{"cells":[{"metadata":{},"cell_type":"markdown","source":"\n\n
\n

This notebook is a work in progress. Trying to cover most common NLP techniques.

\n
"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python Docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load\n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the read-only \"../input/\" directory\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n for filename in filenames:\n print(os.path.join(dirname, filename))\n\n# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using \"Save & Run All\" \n# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session\nimport warnings\nwarnings.filterwarnings('ignore')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import matplotlib.pyplot as plt\nimport seaborn as sns\n\nimport tensorflow as tf\nimport tensorflow_hub as hub\nfrom kaggle_datasets import KaggleDatasets\n\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import metrics\nimport tensorflow as tf\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU,SimpleRNN\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils import np_utils\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\nfrom keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D\nfrom keras.preprocessing import sequence\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.callbacks import EarlyStopping\n\nimport transformers\nfrom transformers import AutoModel, AutoTokenizer, BertTokenizer\nfrom tokenizers import BertWordPieceTokenizer\nfrom transformers import AutoModel, AutoTokenizer, BertTokenizer\nfrom keras import Input, Model\nfrom keras.optimizers import Adam\n\nprint(tf.version.VERSION)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"Files

\njigsaw-toxic-comment-train.csv - data from our first competition. The dataset is made up of English comments from Wikipedia’s talk page edits.
\njigsaw-unintended-bias-train.csv - data from our second competition. This is an expanded version of the Civil Comments dataset with a range of additional labels.
\nsample_submission.csv - a sample submission file in the correct format
\ntest.csv - comments from Wikipedia talk pages in different non-English languages.
\nvalidation.csv - comments from Wikipedia talk pages in different non-English languages.
\njigsaw-toxic-comment-train-processed-seqlen128.csv - training data preprocessed for BERT
\njigsaw-unintended-bias-train-processed-seqlen128.csv - training data preprocessed for BERT
\nvalidation-processed-seqlen128.csv - validation data preprocessed for BERT
\ntest-processed-seqlen128.csv - test data preprocessed for BERT

\nColumns

\nid - identifier within each file.
\ncomment_text - the text of the comment to be classified.
\nlang - the language of the comment.
\ntoxic - whether or not the comment is classified as toxic. (Does not exist in test.csv.)
"},{"metadata":{"trusted":true},"cell_type":"code","source":"# NEW on TPU in TensorFlow 24: shorter cross-compatible TPU/GPU/multi-GPU/cluster-GPU detection code\n\ntry: # detect TPUs\n tpu = tf.distribute.cluster_resolver.TPUClusterResolver.connect() # TPU detection\n strategy = tf.distribute.TPUStrategy(tpu)\nexcept ValueError: # detect GPUs\n strategy = tf.distribute.MirroredStrategy() # for GPU or multi-GPU machines\n #strategy = tf.distribute.get_strategy() # default strategy that works on CPU and single GPU\n #strategy = tf.distribute.experimental.MultiWorkerMirroredStrategy() # for clusters of multi-GPU machines\n\nprint(\"Number of accelerators: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#toxic_comment_train = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\ntoxic_comment_train_processed = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv')\n\n#unintended_bias_train = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train.csv')\nunintended_bias_train_processed = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/jigsaw-unintended-bias-train-processed-seqlen128.csv')\n\n#validation = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\nvalidation_processed = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/validation-processed-seqlen128.csv')\n\n\n#test = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/test.csv')\n#test_processed = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/test-processed-seqlen128.csv')\n\n\n'''\n> test-processed-seqlen128.csv is missing the lang column, so we load test.csv too\n> All other columns are retained in the toxic comment train and unintended bias train datasets, so we need not load unprocessed.\n> All processed datasets have common new column information namely, 'input_word_ids', 'input_mask', 'all_segment_id'\n\n''' \nsample_submission = pd.read_csv('../input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\n\n'''\nSubmission Format being...\n\tid\ttoxic\n0\t0\t0.5\n1\t1\t0.5\n2\t2\t0.5\n3\t3\t0.5\n4\t4\t0.5\n'''\nprocessed_column_names = ['input_word_ids','input_mask','all_segment_id']\ncmn_toxic_params = ['toxic', 'severe_toxic','obscene','threat','insult','identity_hate']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import re\nimport nltk\nfrom nltk.corpus import stopwords\nfrom nltk.tokenize import word_tokenize\nimport spacy\n#nltk.download('stopwords')\n\ndef text_preprocessing(text):\n text = re.sub(r'(@.*?)[\\s]', ' ', text)\n text = re.sub(r'[0-9]+' , '' ,text)\n text = re.sub(r'\\s([@][\\w_-]+)', '', text).strip()\n text = re.sub(r'&', '&', text)\n text = re.sub(r'\\s+', ' ', text).strip()\n text = re.sub(r'https?://\\S+|www\\.\\S+','',text) #hyperlinks\n text = re.sub(r'[!@#$|.\"]','',text) #symbols\n text = text.replace(\"#\" , \" \")\n encoded_string = text.encode(\"ascii\", \"ignore\")\n decode_string = encoded_string.decode()\n return decode_string\n\nsp = spacy.load('en_core_web_sm')\nall_stopwords = sp.Defaults.stop_words\n\ndef without_stopwords(comment):\n comment_tokens_with_sw = word_tokenize(comment)\n tokens_without_sw = [word for word in comment_tokens_with_sw if not word in all_stopwords]\n filtered_sentence = (\" \").join(tokens_without_sw)\n return filtered_sentence\n\ndef plot_accuracy_and_loss_from_history(history):\n color = sns.color_palette()\n acc = history.history['accuracy']\n loss = history.history['loss']\n epochs = range(len(acc))\n sns.lineplot(epochs, acc, label='Training Accuracy')\n sns.lineplot(epochs, loss,label='Training Loss')\n plt.title('Training Accuracy and loss')\n plt.legend()\n plt.figure()\n plt.show()\n \ndef plot_roc_curve_from_prediction(fpr,tpr,roc_auc):\n plt.title('Receiver Operating Characteristic')\n plt.plot(fpr, tpr, 'b', label = 'AUC = %0.2f' % roc_auc)\n plt.legend(loc = 'lower right')\n plt.plot([0, 1], [0, 1],'r--')\n plt.xlim([0, 1])\n plt.ylim([0, 1])\n plt.ylabel('True Positive Rate')\n plt.xlabel('False Positive Rate')\n plt.show()\n \n \ndef do_history_stuff(history):\n plot_accuracy_and_loss_from_history(history)\n \ndef roc_auc(predictions,target):\n fpr, tpr, thresholds = metrics.roc_curve(target, predictions)\n roc_auc = metrics.auc(fpr, tpr)\n plot_roc_curve_from_prediction(fpr,tpr,roc_auc)\n return roc_auc\n\ndef convert_continious_to_discrete_labels(ubtp, labels):\n \n for i in labels:\n ubtp[i] = ubtp[i].apply(lambda x: 1 if x>0 else 0)\n \n return ubtp\n\ndef print_neat_bad_comments(unintended_bias_train_processed, extra_labels):\n print('Percentage of missing values : {}'.format((unintended_bias_train_processed.isna().sum().max()/unintended_bias_train_processed.shape[0])*100))\n value_counts = dict(unintended_bias_train_processed.isna().sum())\n columns_with_nans = [column_name for column_name, value in zip(value_counts.keys(), value_counts.values()) if (value > 0)]\n print(columns_with_nans)\n print(unintended_bias_train_processed.columns)\n '''\n 'publication_id', 'parent_id', 'article_id', 'rating', 'funny', 'wow','sad', 'likes', 'disagree', 'sexual_explicit',\n 'identity_annotator_count', 'toxicity_annotator_count',\n '''\n ubtp = convert_continious_to_discrete_labels(unintended_bias_train_processed,['toxic','obscene','identity_hate','insult','threat'])\n \n ubtp_original_labels = [col for col in ubtp.columns if not col in extra_labels]\n print(ubtp_original_labels)\n\n ubtp['negative_comment_count'] = ubtp['toxic']+ubtp['severe_toxic']+ubtp['obscene']+ubtp['identity_hate']+ubtp['insult']+ubtp['threat']\n neat_comments = ubtp.loc[ubtp['negative_comment_count'] == 0.0,:]\n bad_comments = ubtp.loc[ubtp['negative_comment_count'] != 0,:]\n\n print('Total number of neat comments : {}'.format(neat_comments.shape[0]))\n print('Total number of bad comments : {}'.format(bad_comments.shape[0]))\n\n print('Total number of missing extra labels in neat comments : {}'.format(neat_comments.isna().sum().max()))\n print('Total number of missing parent_id in neat comments : {}'.format(neat_comments.isna().sum().parent_id))\n\n print('Total number of missing extra labels in bad comments : {}'.format(bad_comments.isna().sum().max()))\n print('Total number of missing parent_id in bad comments : {}'.format(bad_comments.isna().sum().parent_id))\n\n print('Number of comments raising a red flag : {}'.format(ubtp.shape[0] - neat_comments.shape[0]))\n \ndef pie_chart_combined_negative_category(toxic_comment_train_processed,cmn_toxic_params):\n toxic_comment_category_value_counts = toxic_comment_train_processed[cmn_toxic_params+['id']].groupby(by=cmn_toxic_params).count().sort_values('id',ascending=False)\n toxic_combination_matrix = pd.DataFrame(toxic_comment_category_value_counts.iloc[1:,0])\n toxic_combination_matrix = toxic_combination_matrix.reset_index()\n tcm = toxic_combination_matrix[cmn_toxic_params].astype('str')\n tcm['comb_str'] = ''\n toxic_combination_matrix['comb_str'] = tcm['toxic'] + tcm['severe_toxic'] + tcm['obscene'] + tcm['threat'] + tcm['insult'] + tcm['identity_hate']\n\n toxic_combination_matrix['percentage'] = toxic_combination_matrix['id']/toxic_combination_matrix['id'].sum()\n fig1, ax1 = plt.subplots(figsize=(10,10))\n explode = (0.1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0)\n ax1.pie(toxic_combination_matrix['percentage'], explode=explode, labels=toxic_combination_matrix['comb_str'],shadow=True, startangle=20)\n ax1.axis('equal') # Equal aspect ratio ensures that pie is drawn as a circle.\n plt.legend(loc='upper left', bbox_to_anchor=(1.05, 1))\n plt.show()\n return toxic_combination_matrix\n \ndef toxic_weight_comparision(toxic_combination_matrix, cmn_toxic_params):\n '''\n I want to allot weights to the severity of the comments\n\n insult = 1\n obscene\t= 2 \n toxic = 3\n severe_toxic = 4\n identity_hate = 5\n threat = 6\n '''\n nmp = toxic_combination_matrix[cmn_toxic_params].astype('int')\n toxic_combination_matrix['weight'] = nmp['toxic']*3 + nmp['severe_toxic']*4 + nmp['obscene']*2 + nmp['threat']*6 + nmp['insult']*1 + nmp['identity_hate']*5\n toxic_combination_matrix['id'] = toxic_combination_matrix['id'].astype('int')\n\n intensity = toxic_combination_matrix.loc[:,['id','weight']].groupby(by='weight').sum()\n plt.figure(figsize=(20,5))\n plt.bar(x=intensity.index.astype('str'),height=intensity['id'])","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Toxic Comment train processed"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(toxic_comment_train_processed.shape) \ntoxic_comment_train_processed.head()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Unintended Bias train processed"},{"metadata":{"trusted":true},"cell_type":"code","source":"print(unintended_bias_train_processed.shape) \nunintended_bias_train_processed.rename(columns={'severe_toxicity': 'severe_toxic', 'identity_attack':'identity_hate'},inplace=True) \nunintended_bias_train_processed.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"extra_labels = ['asian', 'atheist', 'bisexual','black', 'buddhist', 'christian', 'female', 'heterosexual', 'hindu','homosexual_gay_or_lesbian', 'intellectual_or_learning_disability','jewish', 'latino', 'male', 'muslim', 'other_disability','other_gender', 'other_race_or_ethnicity', 'other_religion','other_sexual_orientation', 'physical_disability','psychiatric_or_mental_illness', 'transgender', 'white', 'created_date','publication_id', 'parent_id', 'article_id', 'rating', 'funny', 'wow','sad', 'likes', 'disagree', 'sexual_explicit','identity_annotator_count', 'toxicity_annotator_count','negative_comment_count']\nprint_neat_bad_comments(unintended_bias_train_processed, extra_labels)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Pie Chart Comparision"},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic_combination_matrix = pie_chart_combined_negative_category(toxic_comment_train_processed,cmn_toxic_params)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Toxic custom weight comparision"},{"metadata":{"trusted":true},"cell_type":"code","source":"toxic_weight_comparision(toxic_combination_matrix, cmn_toxic_params)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Peek into a few examples - comment_text"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nRemoving Unnecessary characters\n'''\ndef print_textprocessing_difference(df):\n pd.set_option('display.max_columns',None) \n pd.set_option('display.expand_frame_repr', False)\n pd.set_option('max_colwidth', -1)\n\n df.loc[df['toxic']==1,['id','comment_text']][:3]\n print(df.iloc[12,1])\n print('\\n -------- \\n')\n print(text_preprocessing(df.iloc[12,1]))\n \nprint_textprocessing_difference(toxic_comment_train_processed)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nRemoving Stop Words\n'''\n#toxic_comment_train_processed['comment_text'] = toxic_comment_train_processed['comment_text'].apply(lambda x: without_stopwords(x))\n#ubtp = unintended_bias_train_processed[['comment_text','toxic']]\n#ubtp['comment_text'] = ubtp['comment_text'].apply(lambda x: without_stopwords(x))\n#ubtp = convert_continious_to_discrete_labels(ubtp,['toxic'])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# xubtp = ubtp.drop(columns=[i for i in ubtp.columns if i not in ['comment_text','toxic']])\n# xubtp['toxic'] = xubtp['toxic'].astype(np.uint8)\n# xubtp.memory_usage().sum()\n# xtctp = toxic_comment_train_processed.drop(columns=[i for i in toxic_comment_train_processed.columns if i not in ['comment_text','toxic']])\n# xtctp['toxic'] = xtctp['toxic'].astype(np.uint8)\n# xtctp.memory_usage().sum()\n# train = pd.concat([xtctp[['comment_text','toxic']],xubtp[['comment_text','toxic']]],axis=0)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n#complete_set = int((toxic_comment_train_processed.shape[0] - (toxic_comment_train_processed.shape[0]%8))/8)\n#train = train.loc[:12000,:]\n#train['comment_text'] = train['comment_text'].apply(lambda x:len(str(x).split())).max()\nxtrain = toxic_comment_train_processed.iloc[:12000,1]\nytrain = toxic_comment_train_processed.iloc[:12000,2]\nxvalid = validation_processed.iloc[:4000,1]\nyvalid = validation_processed.iloc[:4000,3]\nxtrain, xvalid, ytrain, yvalid = train_test_split(X,y,stratify=y,random_state=42,train_size=100000,test_size=50000, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"#tokenizer\nmax_len = 1500\ndef regular_tokenizer(xtrain, xvalid):\n token = Tokenizer(num_words=None)\n token.fit_on_texts(list(xtrain) + list(xvalid))\n xtrain_seq = token.texts_to_sequences(xtrain)\n xtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)\n word_index = token.word_index\n print('Total number of unique words : {}'.format(len(token.word_index)))\n \n return xtrain_seq, xtrain_pad, word_index\n\nxtrain_seq, xtrain_pad, word_index = regular_tokenizer(xtrain, xvalid)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# xvalid_seq = token.texts_to_sequences(xvalid)\n# xvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### TPU + Batchsize"},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.backend.clear_session()\nBATCH_SIZE_PER_REPLICA = 60\nGLOBAL_BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n'''\nwe are passing 12000*0.8 -> 9600 data points in the train set if batchsize is 64 then 9600/64 = 150 (8*8)\nGenerally 8 replicas are available so, the provided batch size is split into 8 and processed parallely\nSimilarly 9600/(12*8) -> 100 batches,\n\nWhich is why batch size per replica being 64 throws an error. 9600/(64*8) = 18.75!! \nIt can be 60\n9600/(60*8) = 20\nIt can also be 120\n9600/(120*8) = 10\n'''","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Glove Embeddings"},{"metadata":{"trusted":true},"cell_type":"code","source":"def extract_glove_embedding_matrix(input_file, word_index):\n '''\n !wget http://nlp.stanford.edu/data/glove.6B.zip\n !unzip -q glove.6B.zip\n '''\n embeddings_index = {}\n f = open('../input/glove-embeddings/glove.6B.100d.txt','r',encoding='utf-8')\n '''\n tqdm - is just used to display a progress bar while the loop is in progress\n The following for loop is used to fetch the values of the weights stored in the pre-trained model 'f' into the embeddings_index\n '''\n for line in tqdm(f):\n values = line.split(' ')\n word = values[0]\n coefs = np.asarray([float(val) for val in values[1:]])\n embeddings_index[word] = coefs\n f.close()\n\n print('Found %s word vectors.' % len(embeddings_index))\n '''\n Here we initialize the embedding matrix with the length of our unique word list that is the word_index and 100 columns. \n 100 because we extracted the weights of glove.6B.100d.txt. it would be 300 if we use glove.6B.300d.txt\n The list of all the pretrained glove model can be found it this link -> https://nlp.stanford.edu/projects/glove/\n\n we fill the embedding_matrix with only our unique words under analysis and not all the 4,00,000 word vectors.\n '''\n embedding_matrix = np.zeros((len(word_index) + 1, 100))\n for word, i in tqdm(word_index.items()):\n embedding_vector = embeddings_index.get(word)\n if embedding_vector is not None:\n embedding_matrix[i] = embedding_vector\n \n return embedding_matrix\n \ninput_file = '../input/glove-embeddings/glove.6B.100d.txt'\nembedding_matrix = extract_glove_embedding_matrix(input_file, word_index)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### SimpleRNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"def execute_simplernn():\n with strategy.scope():\n\n model = Sequential()\n model.add(Embedding(len(word_index) + 1,100,weights=[embedding_matrix],input_length=max_len))\n model.add(SimpleRNN(128))\n model.add(Dense(1, activation='sigmoid'))\n model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\n model.summary()\n history = model.fit(xtrain_pad, ytrain, epochs=5, batch_size=GLOBAL_BATCH_SIZE, verbose =1)\n scores = model.predict(xvalid_pad)\n print(\"Auc: %.2f%%\" % (roc_auc(scores,yvalid)))\n do_history_stuff(history)\ntf.keras.backend.clear_session()\nexecute_simplernn()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### LSTM"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\ndef execute_lstm():\n with strategy.scope():\n lstm_model = Sequential()\n '''\n The embedding layer is the input layer and we know that each word has been modified into a vector of length 100\n weights - glove's weights for the unique words in our analysis\n maximum length of each comment is 1500 which we defined earlier itself\n '''\n lstm_model.add(Embedding(len(word_index) + 1,100, weights=[embedding_matrix],input_length=max_len,trainable=False))\n lstm_model.add(LSTM(128, dropout=0.2, recurrent_dropout=0.2))\n lstm_model.add(Dense(1, activation='sigmoid'))\n lstm_model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy']) \n\n lstm_model.summary()\n lstm_history = lstm_model.fit(xtrain_pad, ytrain, epochs=10, batch_size=GLOBAL_BATCH_SIZE, verbose=1)\n lstm_scores = lstm_model.predict(xvalid_pad)\n print(\"Auc: %.2f%%\" % (roc_auc(lstm_scores,yvalid)))\n do_history_stuff(lstm_history)\ntf.keras.backend.clear_session()\nexecute_lstm()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### GRU"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nGRU Model\n'''\ndef execute_gru():\n with strategy.scope():\n gru_model = Sequential()\n gru_model.add(Embedding(len(word_index) + 1,100,weights=[embedding_matrix],input_length=max_len,trainable=False))\n gru_model.add(SpatialDropout1D(0.3))\n gru_model.add(GRU(128))\n gru_model.add(Dense(1, activation='sigmoid'))\n gru_model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy']) \n\n gru_model.summary()\n gru_history = gru_model.fit(xtrain_pad, ytrain, epochs=30, batch_size=GLOBAL_BATCH_SIZE, verbose=1)\n gru_scores = gru_model.predict(xvalid_pad)\n print(\"Auc: %.2f%%\" % (roc_auc(gru_scores,yvalid)))\n do_history_stuff(gru_history)\ntf.keras.backend.clear_session()\nexecute_gru()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Bi-directional RNN"},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nBi-Directional RNN\n'''\ndef execute_bidirectionalrnn():\n with strategy.scope():\n bidirectional_model = Sequential()\n bidirectional_model.add(Embedding(len(word_index) + 1,100,weights=[embedding_matrix],input_length=max_len,trainable=False))\n bidirectional_model.add(Bidirectional(LSTM(128, dropout=0.2, recurrent_dropout=0.2)))\n bidirectional_model.add(Dense(1,activation='sigmoid'))\n bidirectional_model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])\n\n bidirectional_model.summary()\n bidirectional_history = bidirectional_model.fit(xtrain_pad, ytrain, epochs=5, batch_size=GLOBAL_BATCH_SIZE, verbose=1)\n bidirectional_scores = bidirectional_model.predict(xvalid_pad)\n print(\"Auc: %.2f%%\" % (roc_auc(bidirectional_scores,yvalid)))\n do_history_stuff(bidirectional_history)\ntf.keras.backend.clear_session() \nexecute_bidirectionalrnn()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=256, maxlen=192):\n \"\"\"\n Encoder for encoding the text into sequence of integers for BERT Input\n \"\"\"\n tokenizer.enable_truncation(max_length=maxlen) #max_length=maxlen\n tokenizer.enable_padding() #max_length=maxlen\n all_ids = []\n \n for i in tqdm(range(0, len(texts), chunk_size)):\n text_chunk = texts[i:i+chunk_size].tolist()\n encs = tokenizer.encode_batch(text_chunk)\n all_ids.extend([enc.ids for enc in encs])\n \n return np.array(all_ids)\n\ndef bert_encode(texts, tokenizer, max_len=512): \n all_tokens = []\n all_masks = []\n all_segments = []\n \n for text in texts:\n text = tokenizer.tokenize(text)\n text = text[:max_len-2]\n input_sequence = [\"[CLS]\"] + text + [\"[SEP]\"]\n pad_len = max_len - len(input_sequence)\n \n tokens = tokenizer.convert_tokens_to_ids(input_sequence)\n tokens += [0] * pad_len\n pad_masks = [1] * len(input_sequence) + [0] * pad_len\n segment_ids = [0] * max_len\n \n all_tokens.append(tokens)\n all_masks.append(pad_masks)\n all_segments.append(segment_ids)\n \n return np.array(all_tokens), np.array(all_masks), np.array(all_segments)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nLoad Data for bert\n'''\ntrain = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\nvalid = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nDefining Tokenizer\n'''\ntokenizer = transformers.DistilBertTokenizer.from_pretrained('distilbert-base-multilingual-cased')\ntokenizer.save_pretrained('.')\nfast_tokenizer = BertWordPieceTokenizer('vocab.txt', lowercase=False)\n'''\nPerform Encoding\n'''\nx_train = fast_encode(train.comment_text.astype(str), fast_tokenizer, maxlen=192)\nx_valid = fast_encode(valid.comment_text.astype(str), fast_tokenizer, maxlen=192)\ny_train = train.toxic.values\ny_valid = valid.toxic.values\n\n\n'''Data Generator'''\nAUTO = tf.data.experimental.AUTOTUNE\nBATCH_SIZE = GLOBAL_BATCH_SIZE","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (tf.data.Dataset.from_tensor_slices((x_train, y_train)).repeat().shuffle(2048).batch(BATCH_SIZE).prefetch(AUTO))\n\nvalid_dataset = (tf.data.Dataset.from_tensor_slices((x_valid, y_valid)).batch(BATCH_SIZE).cache().prefetch(AUTO))\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"tf.keras.backend.clear_session()\n\ndef build_model(transformer, loss='binary_crossentropy', max_len=512):\n input_word_ids = Input(shape=(max_len,), dtype=tf.int32, name=\"input_word_ids\")\n sequence_output = transformer(input_word_ids)[0]\n cls_token = sequence_output[:, 0, :]\n x = tf.keras.layers.Dropout(0.35)(cls_token)\n out = Dense(1, activation='sigmoid')(x)\n \n model = Model(inputs=input_word_ids, outputs=out)\n model.compile(Adam(lr=3e-5), loss=loss, metrics=[tf.keras.metrics.AUC()])\n \n return model\n\nfrom tensorflow.keras import backend as K\n\ndef focal_loss(gamma=2., alpha=.2):\n def focal_loss_fixed(y_true, y_pred):\n pt_1 = tf.where(tf.equal(y_true, 1), y_pred, tf.ones_like(y_pred))\n pt_0 = tf.where(tf.equal(y_true, 0), y_pred, tf.zeros_like(y_pred))\n return -K.mean(alpha * K.pow(1. - pt_1, gamma) * K.log(pt_1)) - K.mean((1 - alpha) * K.pow(pt_0, gamma) * K.log(1. - pt_0))\n return focal_loss_fixed\n\n\nwith strategy.scope():\n transformer_layer = transformers.TFBertModel.from_pretrained('bert-base-uncased')\n model = build_model(transformer_layer, loss=focal_loss(gamma=1.5), max_len=192)\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"n_steps = xtrain.shape[0] // GLOBAL_BATCH_SIZE\nbert_history = model.fit(train_dataset, epochs = 3, batch_size = GLOBAL_BATCH_SIZE, steps_per_epoch = n_steps)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"'''\nencoding on test to predict and create submission file\n'''\nx_test = fast_encode(test.content.astype(str), fast_tokenizer,maxlen=192)\n\ntest_dataset = (tf.data.Dataset.from_tensor_slices(x_test).batch(BATCH_SIZE))\n\nprediction = model.predict(test_dataset)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"len(prediction)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# train_dataset=(tf.data.Dataset\n# .from_tensor_slices((x_train_d,y_train_d))\n# .repeat()\n# .shuffle(2048)\n# .batch(BATCH_SIZE)\n# .cache()\n# .prefetch(AUTO))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# these are the different token attributes \n \n# TEXT\tLEMMA\tPOS\tTAG\tDEP\tSHAPE\tALPHA\tSTOP\ndetails = {'text': [],\n 'lemma': [],\n 'pos' : [],\n 'tag' : [],\n 'dep' : [],\n 'shape' : [],\n 'alpha' : [],\n 'stop' : []\n }\ndetails_id_map = {}\n\nfor id, comment in zip(train['id'],train['comment_text']):\n details = {'text': [],\n 'lemma': [],\n 'pos' : [],\n 'tag' : [],\n 'dep' : [],\n 'shape' : [],\n 'alpha' : [],\n 'stop' : []\n }\n for token in comment:\n details['text'].append(token.text)\n details['lemma'].append(token.lemma_)\n details['pos'].append(token.pos_)\n details['tag'].append(token.tag_)\n details['dep'].append(token.dep_)\n details['shape'].append(token.shape_)\n details['alpha'].append(token.is_alpha)\n details['stop'].append(token.is_stop)\n \n details_id_map[id] = details","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"nlp(train['comment_text'][0])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# from wordcloud import WordCloud,STOPWORDS\n# wordcloud = WordCloud(stopwords=STOPWORDS,\n# background_color='black',\n# width=3000,\n# height=2500\n# ).generate(commonWord)\n# plt.figure(1,figsize=(12, 12))\n# plt.imshow(wordcloud)\n# plt.axis('off')\n# plt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}