{"cells":[{"metadata":{},"cell_type":"markdown","source":"## Import libraries"},{"metadata":{"_uuid":"8f2839f25d086af736a60e9eeb907d3b93b6e0e5","_cell_guid":"b1076dfc-b9ad-4769-8c92-a6c4dae69d19","trusted":true},"cell_type":"code","source":"import numpy as np\nimport pandas as pd\nimport matplotlib.pyplot as plt\n\nimport math\nimport os\nimport re\nimport time\n\nimport tensorflow as tf\nfrom tensorflow import keras\nfrom tensorflow.keras.layers import Dense,Input,LSTM,Bidirectional,Activation,Conv1D,GRU\nfrom tensorflow.keras.callbacks import Callback\nfrom tensorflow.keras.layers import Dropout,Embedding,GlobalMaxPooling1D, MaxPooling1D, Add, Flatten\nfrom tensorflow.keras.preprocessing import text, sequence\nfrom tensorflow.keras.layers import GlobalAveragePooling1D, GlobalMaxPooling1D, concatenate, SpatialDropout1D\nfrom tensorflow.keras import initializers, regularizers, constraints, optimizers, layers, callbacks\nfrom tensorflow.keras.callbacks import EarlyStopping,ModelCheckpoint\nfrom tensorflow.keras.models import Model\nfrom tensorflow.keras.optimizers import Adam\n\nfrom sklearn.model_selection import train_test_split\nfrom sklearn.metrics import roc_auc_score, roc_curve\n\nfrom kaggle_datasets import KaggleDatasets","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## TPU Configs"},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"AUTO = tf.data.experimental.AUTOTUNE\n\nEPOCHS = 10\nBATCH_SIZE = 16 * strategy.num_replicas_in_sync\nMAX_LEN = 200\n\nMAX_FEATURES = 100000\nEMBED_SIZE = 300","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d629ff2d2480ee46fbb7e2d37f6b5fab8052498a","_cell_guid":"79c7e3d0-c299-4dcb-8224-4455121ee9b0","trusted":true},"cell_type":"code","source":"EMBEDDING_FILE = '../input/glove840b300dtxt/glove.840B.300d.txt'\n\ntrain = pd.read_csv('/kaggle/input/toxic-comment-classification/train.csv')\n\nvalid = pd.read_csv('/kaggle/input/toxic-comment-classification/validation.csv')\n\ntest = pd.read_csv('/kaggle/input/toxic-comment-classification/test.csv')","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"cf43ac37cbd14d8baa088648c2275123550135d6","_cell_guid":"e8bd3575-f711-4ca6-a653-8ec1c74c0204","trusted":true},"cell_type":"code","source":"x_train = train['comment_text'].str.lower()\n\ny_train = train['toxic'].values\n\nx_valid = valid['comment_text'].str.lower()\n\ny_valid = valid['toxic'].values\n\nx_test = test['comment_text'].str.lower()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_train, y_train))\n    .repeat()\n    .shuffle(2048)\n    .batch(BATCH_SIZE)\n    .prefetch(AUTO)\n)\n\nvalid_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices((x_valid, y_valid))\n    .batch(BATCH_SIZE)\n    .cache()\n    .prefetch(AUTO)\n)\n\ntest_dataset = (\n    tf.data.Dataset\n    .from_tensor_slices(x_test)\n    .batch(BATCH_SIZE)\n)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"70c576f3b0afd3e779df184f788107fb51233424","_cell_guid":"3b0804b5-d64e-474e-8252-78daa4a4be62","trusted":true},"cell_type":"code","source":"class RocAucEvaluation(Callback):\n    def __init__(self, validation_data=(), interval=1):\n        super(Callback, self).__init__()\n\n        self.interval = interval\n        self.x_val, self.y_val = validation_data\n\n    def on_epoch_end(self, epoch, logs={}):\n        if epoch % self.interval == 0:\n            y_pred = self.model.predict(self.x_val, verbose=0)\n            score = roc_auc_score(self.y_val, y_pred)\n            print(\"\\n ROC-AUC - epoch: {:d} - score: {:.6f}\".format(epoch+1, score))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Tokenizer"},{"metadata":{"_uuid":"b07e998ccedaf3aaaf4b4e67b207ad5490eb24f7","_cell_guid":"7a665c08-b4a9-4792-b40b-481b3da907e5","trusted":true},"cell_type":"code","source":"%%time\n\ntok=text.Tokenizer(num_words=MAX_FEATURES,lower=True)\ntok.fit_on_texts(list(x_train)+list(x_test))\n\nx_train=tok.texts_to_sequences(x_train)\nx_test=tok.texts_to_sequences(x_test)\n\nx_train=sequence.pad_sequences(x_train,maxlen=MAX_LEN)\nx_test=sequence.pad_sequences(x_test,maxlen=MAX_LEN)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"9488bc9d68dfd1fde1f99d23a9f1ed7b30ceb87f","_cell_guid":"9e57a7cb-c061-4361-bbe2-05c0486a3f18","trusted":true},"cell_type":"code","source":"%%time\n\nembeddings_index = {}\nwith open(EMBEDDING_FILE,encoding='utf8') as f:\n    for line in f:\n        values = line.rstrip().rsplit(' ')\n        word = values[0]\n        coefs = np.asarray(values[1:], dtype='float32')\n        embeddings_index[word] = coefs","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"d56ad119931a971b2588355deb726a045764c9ad","_cell_guid":"e2490100-fc9c-4e46-ae84-7dfa65fcddba","trusted":true},"cell_type":"code","source":"word_index = tok.word_index\n\nnum_words = min(MAX_FEATURES, len(word_index) + 1)\nembedding_matrix = np.zeros((num_words, EMBED_SIZE))\nfor word, i in word_index.items():\n    if i >= MAX_FEATURES:\n        continue\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Train Model"},{"metadata":{"_uuid":"560d3faac051bbb95dae6f1bf7013d52b404533c","_cell_guid":"1a4a1cf3-7faf-4ee4-a72e-a258169778a5","trusted":true},"cell_type":"code","source":"with strategy.scope():\n\n    sequence_input = Input(shape=(MAX_LEN, ))\n    \n    x = Embedding(MAX_FEATURES, EMBED_SIZE, weights=[embedding_matrix],trainable = False)(sequence_input)\n\n    x = SpatialDropout1D(0.1)(x)\n\n    x = Bidirectional(GRU(200, return_sequences=True,dropout=0.1,recurrent_dropout=0.1))(x)\n\n    x = Conv1D(200, kernel_size = 3, padding = \"valid\", kernel_initializer = \"glorot_uniform\")(x)\n\n    avg_pool = GlobalAveragePooling1D()(x)\n    max_pool = GlobalMaxPooling1D()(x)\n\n    x = concatenate([avg_pool, max_pool])\n\n    x = Dense(100, activation='relu')(x)\n    x = Dropout(0.1)(x)\n\n    preds = Dense(1, activation=\"sigmoid\")(x)\n    model = Model(sequence_input, preds)\n    model.compile(loss='binary_crossentropy',optimizer=Adam(lr=1e-5),metrics=['accuracy'])\n\nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"7956b05d34604689b7a10d56a61640091559eda2","_cell_guid":"e1962822-5dfb-4249-a714-ce95346150d4","trusted":true},"cell_type":"code","source":"filepath = \"bilstm_cnn.h5\"\n\ncheckpoint = ModelCheckpoint(filepath, monitor='val_auc', verbose=1, save_best_only=True, save_weights_only=True, mode='max')\n\nearly = EarlyStopping(monitor=\"val_auc\", mode=\"max\", patience=5)","execution_count":null,"outputs":[]},{"metadata":{"_uuid":"f7377e50952ffb6cc14bab3b34442788864eed68","scrolled":false,"_cell_guid":"265115a8-296e-4b67-a6fc-02d0a584b501","trusted":true},"cell_type":"code","source":"%%time\n\nn_steps = x_train.shape[0] // BATCH_SIZE\n\nmodel.fit(train_dataset, steps_per_epoch=n_steps, epochs=EPOCHS, validation_data=valid_dataset, callbacks=[early, checkpoint], verbose=1)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"## Metrics"},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\ny_true = test['toxic'].values\n\ny_pred = []\nfor i in x_test:\n    y_pred.append(model.predict(np.array([i])).tolist()[0][0])\n    \ny_pred = np.array(y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"roc_auc_score(y_true, y_pred)","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"fpr, tpr, thresholds = roc_curve(y_true, y_pred, pos_label=0)\nplt.plot(tpr,fpr)\n\nplt.show()","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}