{"cells":[{"metadata":{},"cell_type":"markdown","source":"# Jigsaw Multilingual Toxic Comment Classification\n#### Members: 邱斯、陳則明、楊淳安、黃亦晨","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%matplotlib inline\n\nfrom tqdm.notebook import tqdm\nfrom sklearn.model_selection import train_test_split\nimport tensorflow as tf\nfrom keras.models import Sequential\nfrom keras.layers.recurrent import GRU,SimpleRNN,LSTM\nfrom keras.layers.core import Dense, Activation, Dropout\nfrom keras.layers.embeddings import Embedding\nfrom keras.layers.normalization import BatchNormalization\nfrom keras.utils import np_utils\nfrom sklearn import preprocessing, decomposition, model_selection, metrics, pipeline\nfrom keras.layers import GlobalMaxPooling1D, Conv1D, MaxPooling1D, Flatten, Bidirectional, SpatialDropout1D\nfrom keras.preprocessing import sequence, text\nfrom keras.callbacks import EarlyStopping\n\nimport sys\nimport os\nimport numpy as np\nimport pandas as pd\nimport IPython\nimport matplotlib.pyplot as plt\nimport seaborn as sns\nfrom plotly import graph_objs as go\nimport plotly.express as px\nimport plotly.figure_factory as ff","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"valid = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation-processed-seqlen128.csv\")\ntrain = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train-processed-seqlen128.csv\")\ntest = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test-processed-seqlen128.csv\")\nsubmit = pd.read_csv(\"/kaggle/input/jigsaw-multilingual-toxic-comment-classification/sample_submission.csv\")\ntrain = train[['id', 'comment_text', 'input_word_ids', 'input_mask','all_segment_id', 'toxic']].iloc[:20000]","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train.info()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"train_distribution = train[\"toxic\"].value_counts().values\nvalid_distribution = valid[\"toxic\"].value_counts().values\n\nnon_toxic = [train_distribution[0] / sum(train_distribution) * 100, valid_distribution[0] / sum(valid_distribution) * 100]\ntoxic = [train_distribution[1] / sum(train_distribution) * 100, valid_distribution[1] / sum(valid_distribution) * 100]\n\nplt.figure(figsize=(9,6))\nplt.bar([0, 1], non_toxic, alpha=.4, color=\"r\", width=0.35, label=\"non-toxic\")\nplt.bar([0.4, 1.4], toxic, alpha=.4, width=0.35, label=\"toxic\")\nplt.xlabel(\"Dataset\")\nplt.ylabel(\"Percentage\")\nplt.xticks([0.2, 1.2], [\"train\", \"valid\"])\nplt.legend(loc=\"upper right\")\n\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"print(f\"train: \\nnon-toxic rate: {train_distribution[0] / sum(train_distribution) * 100: .2f} %\\ntoxic rate: {train_distribution[1] / sum(train_distribution) * 100: .2f} %\")\nprint(f\"valid: \\nnon-toxic rate: {valid_distribution[0] / sum(valid_distribution) * 100: .2f} %\\ntoxic rate: {valid_distribution[1] / sum(valid_distribution) * 100: .2f} %\")","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"lang = valid[\"lang\"].value_counts()\n\nplt.figure(figsize=(9, 6))\nplt.xlabel(\"Lang\")\nplt.ylabel(\"Num\")\nplt.xticks([0.2, 0.6, 1], [\"tr\", \"es\", \"it\"])\nplt.bar([0.2, 0.6, 1], lang, color=\"purple\", width=0.28, alpha=.4)\nplt.show()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# TPU","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"# Detect hardware, return appropriate distribution strategy\ntry:\n    # TPU detection. No parameters necessary if TPU_NAME environment variable is\n    # set: this is always the case on Kaggle.\n    tpu = tf.distribute.cluster_resolver.TPUClusterResolver()\n    print('Running on TPU ', tpu.master())\nexcept ValueError:\n    tpu = None\n\nif tpu:\n    tf.config.experimental_connect_to_cluster(tpu)\n    tf.tpu.experimental.initialize_tpu_system(tpu)\n    strategy = tf.distribute.experimental.TPUStrategy(tpu)\nelse:\n    # Default distribution strategy in Tensorflow. Works on CPU and single GPU.\n    strategy = tf.distribute.get_strategy()\n\nprint(\"REPLICAS: \", strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# 讀入資料\n1. 從競賽中取出未經預處理的訓練資料 `jigsaw-toxic-comment-train.csv`\n2. 取出驗證資料 `validation.csv` 和測試資料 `test.csv`\n3. 丟棄不需要的欄位，保留 `[id, comment_text, toxic]`，comment_text 代表 twitter 的留言，toxic 是 1 表示惡意，0 表示安全","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/jigsaw-toxic-comment-train.csv')\nvalidation = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/validation.csv')\ntest = pd.read_csv('/kaggle/input/jigsaw-multilingual-toxic-comment-classification/test.csv')\n\ntrain.drop(['severe_toxic','obscene','threat','insult','identity_hate'], axis=1, inplace=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"1. 從原始 223549 筆訓練資料中讀取出 25000 筆做為我們的訓練資料\n\n從中我們找到最大長度的句子是 - 1403","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"train = train.loc[:25000-1,:]\nprint(f\"訓練資料總數：{train.shape[0]}\\n句子最大長度：{train['comment_text'].apply(lambda x:len(str(x).split())).max()}\")","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"定義本次競賽中作為評估的函式 **roc_auc**，以下為所需的兩個輸入：\n- model 預測出來的 predictions\n- 資料中的標準答案","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"def roc_auc(predictions,target):\n  \n    fpr, tpr, thresholds = metrics.roc_curve(target, predictions)\n    roc_auc = metrics.auc(fpr, tpr)\n    \n    return roc_auc","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"將 25000 筆資料分割為 `train` 跟 `valid` (0.2)\n\n$ 25000\\times0.2=5000 $ (Validation data)\n\n$ 25000-5000=20000 $ (Training data)\n\n之後取出測試資料的 twitter 給 `xtset`","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#取出train和valid所需資料\n#將train中不必要的欄位drop掉\nxtrain, xvalid, ytrain, yvalid = train_test_split(train.comment_text.values, train.toxic.values, \n                                                  stratify=train.toxic.values, \n                                                  random_state=42, \n                                                  test_size=0.2, shuffle=True)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Tokenizer\n1. 將每一筆資料的評論部分做斷詞，得到 token sequences \n2. 將每筆資料調整成同一大小 (128字) --> zero padding\n3. 最後 3 個資料集 (train, valid, test) 的長度分別是 (20000, 5000, 63812)，每個句子因為 padding，所以都是 128\n4. word_index 紀錄著每一個字對應的 id","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"token = text.Tokenizer(num_words=None)\nmax_len = 128\n\ntoken.fit_on_texts(list(xtrain)+ list(xvalid)) #+ list(xvalid)+list(test)\nxtrain_seq = token.texts_to_sequences(xtrain)\nxvalid_seq = token.texts_to_sequences(xvalid)\n\nxtrain_pad = sequence.pad_sequences(xtrain_seq, maxlen=max_len)\nxvalid_pad = sequence.pad_sequences(xvalid_seq, maxlen=max_len)\n\nword_index = token.word_index\nprint(xtrain_pad.shape, xvalid_pad.shape)\n","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Word Embedding\n1. 載入 GloVe 的詞量中，將之整理成一個 dict `embeddings_index`\n2. 每一行的第一個字是一個單字，剩下是他對應的向量\n3. 我們最後可以看到整個 GloVe 有 2196017 個單字存在","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"embeddings_index = {}\nf = open('/kaggle/input/glove840b300dtxt/glove.840B.300d.txt','r',encoding='utf-8')\nfor line in tqdm(f):\n    values = line.split(' ')\n    word = values[0]\n    coefs = np.asarray([float(val) for val in values[1:]])\n    embeddings_index[word] = coefs\nf.close()\n\nprint('Found %s word vectors.' % len(embeddings_index))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"embedding_matrix = np.zeros((len(word_index) + 1, 300))\nfor word, i in tqdm(word_index.items()):\n    embedding_vector = embeddings_index.get(word)\n    if embedding_vector is not None:\n        embedding_matrix[i] = embedding_vector","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LSTM\n1. 將第一層設定為 300 維的 Embedding layer\n2. 三層 LSTM 層，各 300 個神經元\n3. 設定 drop out，避免 overfitting\n3. 將輸出使用 sigmoid 拉到 0~1 之間\n4. Compiling，因為是二分類別所以使用 binary_crossentropy 當 loss function，Adam 當我們 optimization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    \n    model = Sequential()\n    model.add(Embedding(len(word_index) + 1,\n                     300,\n                     weights=[embedding_matrix],\n                     input_length=max_len,\n                     trainable=False))\n\n    model.add(LSTM(300, dropout=0.3, recurrent_dropout=0.3,return_sequences=True))\n    model.add(LSTM(300, dropout=0.3, recurrent_dropout=0.3,return_sequences=True))\n    model.add(LSTM(300, dropout=0.3, recurrent_dropout=0.3))\n    model.add(Dense(1, activation='sigmoid'))\n    \n    model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n    \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\n\nmodel.fit(xtrain_pad, ytrain, epochs=5, batch_size=48*strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# LSTM模型在 valid 的表現 ","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"scores = model.predict(xvalid_pad)\nprint(\"Auc: %.2f%%\" % (roc_auc(scores,yvalid)))","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# GRU\n1. 將第一層設定為 300 維的 Embedding layer\n2. 三層 GRU 層，各 300 個神經元\n3. 設定 drop out，避免 overfitting\n4. 將輸出使用 sigmoid 拉到 0~1 之間\n5. Compiling，因為是二分類別所以使用 binary_crossentropy 當 loss function，Adam 當我們 optimization","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"%%time\nwith strategy.scope():\n    # GRU with glove embeddings and two dense layers\n     model = Sequential()\n     model.add(Embedding(len(word_index) + 1,\n                     300,\n                     weights=[embedding_matrix],\n                     input_length=max_len,\n                     trainable=False))\n     model.add(SpatialDropout1D(0.3))\n     model.add(GRU(300,return_sequences=True))\n     model.add(Dropout(0.3))\n     model.add(GRU(300,return_sequences=True))\n     model.add(Dropout(0.3))\n     model.add(GRU(300,return_sequences=False))\n     model.add(Dropout(0.3))\n     model.add(Dense(1, activation='sigmoid'))\n\n     model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy'])   \n    \nmodel.summary()","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# Train","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#GRU模型訓練\nmodel.fit(xtrain_pad, ytrain, epochs=5, batch_size=64*strategy.num_replicas_in_sync)","execution_count":null,"outputs":[]},{"metadata":{},"cell_type":"markdown","source":"# GRU模型在 valid 的表現","execution_count":null},{"metadata":{"trusted":true},"cell_type":"code","source":"#使用valid測試資料的AUC結果\nscores = model.predict(xvalid_pad)\nprint(\"Auc: %.2f%%\" % (roc_auc(scores,yvalid)))","execution_count":null,"outputs":[]},{"metadata":{"trusted":true},"cell_type":"code","source":"","execution_count":null,"outputs":[]}],"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"pygments_lexer":"ipython3","nbconvert_exporter":"python","version":"3.6.4","file_extension":".py","codemirror_mode":{"name":"ipython","version":3},"name":"python","mimetype":"text/x-python"}},"nbformat":4,"nbformat_minor":4}