{"cells":[{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"# This Python 3 environment comes with many helpful analytics libraries installed\n# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n# For example, here's several helpful packages to load in \n\nimport numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n\n# Input data files are available in the \"../input/\" directory.\n# For example, running this (by clicking run or pressing Shift+Enter) will list all files under the input directory\n\nimport os\nfor dirname, _, filenames in os.walk('/kaggle/input'):\n    for filename in filenames:\n        print(os.path.join(dirname, filename))\n\n# Any results you write to the current directory are saved as output.","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"It only takes one toxic comment to sour an online discussion. The Conversation AI team, a research initiative founded by Jigsaw and Google, builds technology to protect voices in conversation. A main area of focus is machine learning models that can identify toxicity in online conversations, where toxicity is defined as anything rude, disrespectful or otherwise likely to make someone leave a discussion. If these toxic contributions can be identified, we could have a safer, more collaborative internet.\n\nIn the previous 2018 Toxic Comment Classification Challenge, Kagglers built multi-headed models to recognize toxicity and several subtypes of toxicity. In 2019, in the Unintended Bias in Toxicity Classification Challenge, you worked to build toxicity models that operate fairly across a diverse range of conversations. This year, we're taking advantage of Kaggle's new TPU support and challenging you to build multilingual models with English-only training data.\n\nJigsaw's API, Perspective, serves toxicity models and others in a growing set of languages (see our documentation for the full list). Over the past year, the field has seen impressive multilingual capabilities from the latest model innovations, including few- and zero-shot learning. We're excited to learn whether these results \"translate\" (pun intended!) to toxicity classification. Your training data will be the English data provided for our previous two competitions and your test data will be Wikipedia talk page comments in several different languages.\n\nAs our computing resources and modeling capabilities grow, so does our potential to support healthy conversations across the globe. Develop strategies to build effective multilingual models and you'll help Conversation AI and the entire industry realize that potential.\n\nDisclaimer: The dataset for this competition contains text that may be considered profane, vulgar, or offensive."},{"metadata":{"trusted":true},"cell_type":"code","source":"import warnings\nwarnings.filterwarnings(\"ignore\")\n\nimport os\nimport gc\nimport re\nimport folium\n#import textstat\nfrom scipy import stats\nfrom colorama import Fore, Back, Style, init\n\nimport math\nimport numpy as np\nimport scipy as sp\nimport pandas as pd\n\nimport random\nimport networkx as nx\nfrom pandas import Timestamp\n\nfrom PIL import Image\nfrom IPython.display import SVG\nfrom keras.utils import model_to_dot\n\nimport requests\nfrom IPython.display import HTML\n\nimport seaborn as sns\nfrom tqdm import tqdm\nimport matplotlib.cm as cm\nimport matplotlib.pyplot as plt\n\ntqdm.pandas()\n\nimport plotly.express as px\nimport plotly.graph_objects as go\nimport plotly.figure_factory as ff\nfrom plotly.subplots import make_subplots\n\n#import transformers\nimport tensorflow as tf\n\nfrom tensorflow.keras.callbacks import Callback\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom tensorflow.keras.callbacks import ModelCheckpoint, ReduceLROnPlateau, CSVLogger\n\nfrom tensorflow.keras.models import Model\n#from kaggle_datasets import KaggleDatasets\n#from tensorflow.keras.optimizers import Adam\n#from tokenizers import BertWordPieceTokenizer\nfrom tensorflow.keras.layers import Dense, Input, Dropout, Embedding\nfrom tensorflow.keras.layers import LSTM, GRU, Conv1D, SpatialDropout1D\n\nfrom tensorflow.keras import layers\nfrom tensorflow.keras import optimizers\nfrom tensorflow.keras import activations\nfrom tensorflow.keras import constraints\nfrom tensorflow.keras import initializers\nfrom tensorflow.keras import regularizers\n\nimport tensorflow.keras.backend as K\nfrom tensorflow.keras.layers import *\nfrom tensorflow.keras.optimizers import *\nfrom tensorflow.keras.activations import *\nfrom tensorflow.keras.constraints import *\nfrom tensorflow.keras.initializers import *\nfrom tensorflow.keras.regularizers import *\n\nfrom sklearn import metrics\nfrom sklearn.utils import shuffle\nfrom gensim.models import Word2Vec\nfrom sklearn.cluster import KMeans\nfrom sklearn.decomposition import PCA\nfrom sklearn.feature_extraction.text import TfidfVectorizer,\\\n                                            CountVectorizer,\\\n                                            HashingVectorizer\n\nfrom nltk.stem.wordnet import WordNetLemmatizer \nfrom nltk.tokenize import word_tokenize\nfrom nltk.tokenize import TweetTokenizer  \n\nimport nltk\nfrom textblob import TextBlob\n\nfrom nltk.corpus import wordnet\nfrom nltk.corpus import stopwords\n#from googletrans import Translator\nfrom nltk import WordNetLemmatizer\n#from polyglot.detect import Detector\nfrom nltk.stem import WordNetLemmatizer\nfrom wordcloud import WordCloud, STOPWORDS\nfrom nltk.sentiment.vader import SentimentIntensityAnalyzer\n\nstopword=set(STOPWORDS)\n\nlem = WordNetLemmatizer()\ntokenizer=TweetTokenizer()\n\nnp.random.seed(0)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"train_data = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\nval_data = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest_data = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"test_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val_data.head()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.columns","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.shape","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.isnull().sum()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**TOXIC**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data[\"toxic\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# making a donut chart to represent share of each.\n\nsize = [202165, 21384]\ncolors = ['pink', 'lightblue']\nlabels = \"0\",\"1\"\n\nmy_circle = plt.Circle((0, 0), 0.7, color = 'white')\n\nplt.rcParams['figure.figsize'] = (10, 10)\nplt.pie(size, colors = colors, labels = labels, autopct = '%.2f%%')\nplt.axis('off')\nplt.title('A Pie Chart Representing share of Toxic Comments', fontsize = 30)\np = plt.gcf()\nplt.gca().add_artist(my_circle)\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Words cloud of Negative toxic sentiments**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\nNegative_sentiments = \" \".join([text for text in train_data['comment_text'][train_data['toxic'] == 1]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'lightgreen', stopwords = stopwords, width = 1200, height = 800).generate(Negative_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common Negative Toxic Words', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Words cloud of Positive sentiments**"},{"metadata":{"trusted":true},"cell_type":"code","source":"# making Words cloud for the postive sentiments\n\nPositive_sentiments = \" \".join([text for text in train_data['comment_text'][train_data['toxic'] == 0]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'pink', stopwords = stopwords, width = 1200, height = 800).generate(Positive_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common Positive Words', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**SEVERE TOXIC**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data[\"severe_toxic\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# making a donut chart \n\nsize = [221587, 1962]\ncolors = ['yellow', 'orange']\nlabels = \"0\",\"1\"\n\nmy_circle = plt.Circle((0, 0), 0.7, color = 'white')\n\nplt.rcParams['figure.figsize'] = (10, 10)\nplt.pie(size, colors = colors, labels = labels, autopct = '%.2f%%')\nplt.axis('off')\nplt.title('A Pie Chart Representing the Share of Toxic', fontsize = 30)\np = plt.gcf()\nplt.gca().add_artist(my_circle)\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":" **Words cloud for Negative severe_toxic sentiments**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nNegative_sentiments = \" \".join([text for text in train_data['comment_text'][train_data['severe_toxic'] == 1]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'cyan', stopwords = stopwords, width = 1200, height = 800).generate(Negative_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common Negative severe toxic words', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nPositive_sentiments = \" \".join([text for text in train_data['comment_text'][train_data['severe_toxic'] == 0]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'yellow', stopwords = stopwords, width = 1200, height = 800).generate(Positive_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common Positive Words', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**\nObscene\n**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data[\"obscene\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# making a donut chart to represent share of each ratings\n\nsize = [211409, 12140]\ncolors = ['cyan', 'maroon']\nlabels = \"0\",\"1\"\n\nmy_circle = plt.Circle((0, 0), 0.7, color = 'white')\n\nplt.rcParams['figure.figsize'] = (10, 10)\nplt.pie(size, colors = colors, labels = labels, autopct = '%.2f%%')\nplt.axis('off')\nplt.title('A Pie Chart Representing the Share of Toxic', fontsize = 30)\np = plt.gcf()\nplt.gca().add_artist(my_circle)\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Words cloud for the Negative Obscene words**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nNegative_sentiments = \" \".join([text for text in train_data['comment_text'][train_data[\"obscene\"] == 1]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'pink', stopwords = stopwords, width = 1200, height = 800).generate(Negative_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common Negative Obscene words', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nPositive_sentiments = \" \".join([text for text in train_data['comment_text'][train_data[\"obscene\"] == 0]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'grey', stopwords = stopwords, width = 1200, height = 800).generate(Positive_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common Words n Positive', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**THREAT**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data[\"threat\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# making a donut chart to represent share of each ratings\n\nsize = [222860,  689]\ncolors = ['lightgreen', 'yellow']\nlabels = \"0\",\"1\"\n\nmy_circle = plt.Circle((0, 0), 0.7, color = 'white')\n\nplt.rcParams['figure.figsize'] = (10, 10)\nplt.pie(size, colors = colors, labels = labels, autopct = '%.2f%%')\nplt.axis('off')\nplt.title('A Pie Chart Representing the Share of Toxic', fontsize = 30)\np = plt.gcf()\nplt.gca().add_artist(my_circle)\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Words cloud for the Negative THreat words**"},{"metadata":{"trusted":true},"cell_type":"code","source":"Negative_sentiments = \" \".join([text for text in train_data['comment_text'][train_data[\"threat\"] == 1]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'blue', stopwords = stopwords, width = 1200, height = 800).generate(Negative_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common Negative threat Words', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"Positive_sentiments = \" \".join([text for text in train_data['comment_text'][train_data[\"obscene\"] == 0]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'black', stopwords = stopwords, width = 1200, height = 800).generate(Positive_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common Words n Positive', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**IDENTITY HATE\n**"},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data[\"identity_hate\"].value_counts()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# making a donut chart to represent share of each ratings\n\nsize = [221432,   2117]\ncolors = ['yellow', 'magenta']\nlabels = \"0\",\"1\"\n\nmy_circle = plt.Circle((0, 0), 0.7, color = 'white')\n\nplt.rcParams['figure.figsize'] = (10, 10)\nplt.pie(size, colors = colors, labels = labels, autopct = '%.2f%%')\nplt.axis('off')\nplt.title('A Pie Chart Representing the Share of Toxic', fontsize = 30)\np = plt.gcf()\nplt.gca().add_artist(my_circle)\nplt.legend()\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Words cloud for NEGATIVE HATE WORDS**"},{"metadata":{"trusted":true},"cell_type":"code","source":"\n\nNegative_sentiments = \" \".join([text for text in train_data['comment_text'][train_data['identity_hate'] == 1]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'purple', stopwords = stopwords, width = 1200, height = 800).generate(Negative_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common  NEGATIVE HATE WORDS', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Postive sentiments\n\nPositive_sentiments = \" \".join([text for text in train_data['comment_text'][train_data['identity_hate'] == 0]])\n\nfrom wordcloud import WordCloud\nfrom wordcloud import STOPWORDS\n\nstopwords = set(STOPWORDS)\nwordcloud = WordCloud(background_color = 'lightblue', stopwords = stopwords, width = 1200, height = 800).generate(Positive_sentiments)\n\nplt.rcParams['figure.figsize'] = (15, 15)\nplt.title('Most Common Words n Positive_sentiments', fontsize = 30)\nprint(wordcloud)\nplt.axis('off')\nplt.imshow(wordcloud)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"**Distribution of comment words**"},{"metadata":{"trusted":true},"cell_type":"code","source":"def new_len(x):\n    if type(x) is str:\n        return len(x.split())\n    else:\n        return 0\n\ntrain_data[\"comment_words\"] = train_data[\"comment_text\"].apply(new_len)\nnums = train_data.query(\"comment_words != 0 and comment_words < 200\").sample(frac=0.1)[\"comment_words\"]\nfig = ff.create_distplot(hist_data=[nums],\n                         group_labels=[\"All comments\"],\n                         colors=[\"black\"])\n\nfig.update_layout(title_text=\"Comment words\", xaxis_title=\"Comment words\", template=\"simple_white\", showlegend=False)\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = go.Figure(data=[\n    go.Pie(labels=train_data.columns[2:7],\n           values=train_data.iloc[:, 2:7].sum().values, marker=dict(colors=px.colors.qualitative.Plotly))\n])\nfig.update_traces(textposition='outside', textfont=dict(color=\"black\"))\nfig.update_layout(title_text=\"Pie chart of labels\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fig = go.Figure(data=[\n    go.Bar(y=train_data.columns[2:7],\n           x=train_data.iloc[:, 2:7].sum().values, marker=dict(color=px.colors.qualitative.Plotly))\n])\n\nfig.data[0].marker.line.color = 'rgb(0, 0, 0)'\nfig.data[0].marker.line.width = 0.75\nfig.update_traces(orientation=\"h\")\nfig.update_layout(title_text=\"Bar chart of labels\", template=\"plotly_white\")\nfig.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"val = val_data\ntrain = train_data\n\ndef clean(text):\n    text = text.fillna(\"fillna\").str.lower()\n    text = text.map(lambda x: re.sub('\\\\n',' ',str(x)))\n    text = text.map(lambda x: re.sub(\"\\[\\[User.*\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\\.\\d{1,3}\",'',str(x)))\n    text = text.map(lambda x: re.sub(\"\\(http://.*?\\s\\(http://.*\\)\",'',str(x)))\n    return text\n\nval[\"comment_text\"] = clean(val[\"comment_text\"])\ntest_data[\"content\"] = clean(test_data[\"content\"])\ntrain[\"comment_text\"] = clean(train[\"comment_text\"])","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"class RocAucEvaluation(Callback):\n    def __init__(self, validation_data=(), interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.X_val, self.y_val = validation_data\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict(self.X_val, verbose=0)\n            score = roc_auc_score(self.y_val, y_pred)\n            print(\"\\n ROC-AUC - epoch: {:d} - score: {:.6f}\".format(epoch+1, score))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def fast_encode(texts, tokenizer, chunk_size=240, maxlen=512):\n    tokenizer.enable_truncation(max_length=maxlen)\n    tokenizer.enable_padding(max_length=maxlen)\n    all_ids = []\n    \n    for i in range(0, len(texts), chunk_size):\n        text_chunk = texts[i:i+chunk_size].tolist()\n        encs = tokenizer.encode_batch(text_chunk)\n        all_ids.extend([enc.ids for enc in encs])\n    \n    return np.array(all_ids)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"from tokenizers import Tokenizer, Encoding\nfrom tokenizers import decoders\nfrom tokenizers import models\nfrom tokenizers import normalizers\nfrom tokenizers import pre_tokenizers\nfrom tokenizers import processors\nfrom tokenizers import trainers\nfrom tokenizers import BertWordPieceTokenizer","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data.drop(['severe_toxic','obscene','threat','insult','identity_hate'],axis=1,inplace=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_data['comment_text'].apply(lambda x:len(str(x).split())).max()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"import numpy as np # linear algebra\nimport pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\nfrom tqdm import tqdm\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import LSTM, GRU,SimpleRNN\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils import np_utils\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\nfrom keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D\nfrom keras.preprocessing import sequence, text\nfrom keras.callbacks import EarlyStopping\n\nxtrain, xvalid, ytrain, yvalid = train_test_split(train_data.comment_text.values, train.toxic.values, \n                                                  stratify=train.toxic.values, \n                                                  random_state=42, \n                                                  test_size=0.2, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"# using keras tokenizer here\ntoken = text.Tokenizer(num_words=None)\nmax_len = 1500\n\ntoken.fit_on_texts(list(xtrain) + list(xvalid))\nxtrain_seq = token.texts_to_sequences(xtrain)\nxvalid_seq = token.texts_to_sequences(xvalid)\n\n#zero pad the sequences\nxtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)\nxvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)\n\nword_index = token.word_index","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    # A simpleRNN without any pretrained embeddings and one dense layer\n    model = Sequential()\n    model.add(Embedding(len(word_index) + 1,\n                     300,\n                     input_length=max_len))\n    model.add(SimpleRNN(100))\n    model.add(Dense(1, activation='sigmoid'))\n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"model.fit(xtrain_pad, ytrain, batch_size=64*strategy.num_replicas_in_sync) ","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"def roc_auc(predictions,target):\n    '''\n    This methods returns the AUC Score when given the Predictions\n    and Labels\n    '''\n    \n    fpr, tpr, thresholds = metrics.roc_curve(target, predictions)\n    roc_auc = metrics.auc(fpr, tpr)\n    return roc_auc","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"\nscores = model.predict(xvalid_pad)\nprint(\"Auc: %.2f%%\" % (roc_auc(scores,yvalid)))","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}