{"cells":[{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nimport tensorflow as tf\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\n\nfrom keras.preprocessing import sequence,text\nfrom keras.models import Sequential\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.recurrent import LSTM, GRU,SimpleRNN\nfrom keras.layers.core import Dense, Activation, Dropout\nimport re,string\n#!pip install pyspellchecker\n#from spellchecker import SpellChecker\nfrom nltk.tokenize import word_tokenize\nfrom nltk.corpus import stopwords\nstop=set(stopwords.words('english'))\nfrom keras.preprocessing.text import Tokenizer\nfrom keras.preprocessing.sequence import pad_sequences\n\nimport matplotlib.pyplot as plt\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)\n\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Data Loading and EDA","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv\")\n\ntest_data = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv\")\ntest_data.columns = ['id','comment_text','lang']\nvalidation_data = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv\")\nsub = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nDrop the other columns in the training data \n\"\"\"\n\ntrain_data.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True)\nlen(train_data)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nMaximum Number of words in Comments\n\"\"\"\ntrain_data['comment_text'].apply(lambda x: len(str(x).split())).max()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Data Cleaning","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def remove_URL(text):\n    url = re.compile(r'https?://\\S+|www\\.\\S+')\n    return url.sub(r'',text)\n\ndef remove_html(text):\n    html=re.compile(r'<.*?>')\n    return html.sub(r'',text)\n\ndef remove_emoji(text):\n    emoji_pattern = re.compile(\"[\"\n                           u\"\\U0001F600-\\U0001F64F\"  # emoticons\n                           u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n                           u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n                           u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n                           u\"\\U00002702-\\U000027B0\"\n                           u\"\\U000024C2-\\U0001F251\"\n                           \"]+\", flags=re.UNICODE)\n    return emoji_pattern.sub(r'', text)\n\ndef remove_punct(text):\n    table=str.maketrans('','',string.punctuation)\n    return text.translate(table)\n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ntrain_data['text']=train_data['comment_text'].apply(lambda x : remove_URL(x))\ntrain_data['text']=train_data['comment_text'].apply(lambda x : remove_html(x))\ntrain_data['text']=train_data['comment_text'].apply(lambda x: remove_emoji(x))\ntrain_data['text']=train_data['comment_text'].apply(lambda x : remove_punct(x))\n\ntest_data['text']=test_data['content'].apply(lambda x : remove_URL(x))\ntest_data['text']=test_data['content'].apply(lambda x : remove_html(x))\ntest_data['text']=test_data['content'].apply(lambda x: remove_emoji(x))\ntest_data['text']=test_data['content'].apply(lambda x : remove_punct(x))\ndf=pd.concat([train_data,test_data])\ndf['text']=df['comment_text'].apply(lambda x : remove_URL(x))\ndf['text']=df['comment_text'].apply(lambda x : remove_html(x))\ndf['text']=df['comment_text'].apply(lambda x: remove_emoji(x))\ndf['text']=df['comment_text'].apply(lambda x : remove_punct(x))\n\n\n\n\"\"\"\n\nfor dataset in [train_data, test_data]:\n    \n    dataset['text']=dataset['comment_text'].apply(lambda x : remove_URL(x))\n    dataset['text']=dataset['comment_text'].apply(lambda x : remove_html(x))\n    dataset['text']=dataset['comment_text'].apply(lambda x: remove_emoji(x))\n    dataset['text']=dataset['comment_text'].apply(lambda x : remove_punct(x))\n\n    \n    \n\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nY = train_data['toxic']","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nTokenization\n\"\"\"\nmax_length = 1500\ntokenizer = Tokenizer()\ntokenizer.fit_on_texts(train_data['comment_text'])\n\ntrain_tokenized = tokenizer.texts_to_sequences(train_data['comment_text'])\ntest_tokenized = tokenizer.texts_to_sequences(test_data['comment_text'])\n\nX = pad_sequences(train_tokenized, maxlen=max_length)\nX_ = pad_sequences(test_tokenized, maxlen=max_length)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"word_index = tokenizer.word_index\nword_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Load the GloVe vectors in a dictionary:\n\n\n \nembeddings_index = {}\nf = open('/kaggle/input/glove840b300dtxt/glove.840B.300d.txt','r',encoding='utf-8')\nfor line in tqdm(f):\n    values = line.split(' ')\n    word = values[0]\n    coefs = np.asarray([float(val) for val in values[1:]])\n    embeddings_index[word] = coefs\nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))\n\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"num_words=len(word_index)+1\nembedding_matrix=np.zeros((num_words,100))\n\n# create an embedding matrix for the words we have in the dataset\nembedding_matrix = np.zeros((len(word_index) + 1, 300))\nfor word, i in tqdm(word_index.items()):\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model = Sequential()\nmodel.add(Embedding(len(word_index) + 1,\n                 300,\n                 weights=[embedding_matrix],\n                 input_length=1500,\n                 trainable=False))\n\nmodel.add(LSTM(100, dropout=0.3, recurrent_dropout=0.3))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(X, Y, batch_size=1024, epochs =2)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"pred = model.predict(X_)\ntemp = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv')\ntemp['toxic'] = pred\ntemp.to_csv('submission.csv', index=False)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"      \ndef roc_auc(predictions,target):\n    '''\n    This methods returns the AUC Score when given the Predictions\n    and Labels\n    '''\n    \n    fpr, tpr, thresholds = metrics.roc_curve(target, predictions)\n    roc_auc = metrics.auc(fpr, tpr)\n    return roc_auc\n\"\"\"\n ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nData Preparation\n\nxtrain, xvalid, ytrain, yvalid = train_test_split(train_data.comment_text.values, train_data.toxic.values, \n                                                  stratify=train_data.toxic.values, \n                                                  random_state=42, \n                                                  test_size=0.2, shuffle=True)\nxtrain\n\"\"\"","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Recurrent Neural Network Model","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ntoken = text.Tokenizer(num_words=None)\nmax_len = 1500\ntoken.fit_on_texts(list(xtrain) + list(xvalid))\nprint(token.word_index)\nxtrain_seq = token.texts_to_sequences(xtrain)\nxvalid_seq = token.texts_to_sequences(xvalid)\n\nxtrain_pad = sequence.pad_sequences(xtrain_seq,max_len)\nxvalid_pad = sequence.pad_sequences(xvalid_seq,max_len)\n\nword_index = token.word_index\n\"\"\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nmodel = Sequential()\nmodel.add(Embedding(len(word_index)+1, 300, input_length=max_len))\nmodel.add(SimpleRNN(100))\nmodel.add(Dense(1, activation='sigmoid'))\nmodel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n\"\"\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nmodel.fit(xtrain_pad, ytrain, nb_epoch=2, batch_size=64) \nscores = model.predict(xvalid_pad)\nprint(\"Auc: %.2f%%\" % (roc_auc(scores,yvalid)))\n\"\"\"\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"### Word Embeddings - LSTM \n","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\ntoken = text.Tokenizer(num_words=None)\nmax_len = 1500\ntoken.fit_on_texts(list(xtrain) + list(xvalid))\nprint(token.word_index)\nxtrain_seq = token.texts_to_sequences(xtrain)\nxvalid_seq = token.texts_to_sequences(xvalid)\n\nxtrain_pad = sequence.pad_sequences(xtrain_seq,max_len)\nxvalid_pad = sequence.pad_sequences(xvalid_seq,max_len)\n\nword_index = token.word_index\n\"\"\"\n\n\"\"\"\nGlove for Vectorization\n\ndef create_corpus(df):\n    corpus=[]\n    for tweet in tqdm(df['comment_text']):\n        #print(tweet)\n        words=[word.lower() for word in word_tokenize(tweet) if((word.isalpha()==1) & (word not in stop))]\n        #print(words)\n        corpus.append(words)\n    return corpus\n\ncorpus=create_corpus(df)\n\n\"\"\"\n","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\"\"\"\nMAX_LEN = 1500\n\ntokenizer_obj=Tokenizer()\ntokenizer_obj.fit_on_texts(corpus)\nsequences=tokenizer_obj.texts_to_sequences(corpus)\ntweet_pad=pad_sequences(sequences,maxlen=MAX_LEN,truncating='post',padding='post')\n\nword_index=tokenizer_obj.word_index\nprint('Number of unique words:',len(word_index))\n\n\"\"\"\n\n","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}